test_sparse_basic.c revision 311041
1/*- 2 * Copyright (c) 2010-2012 Michihiro NAKAJIMA 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25#include "test.h" 26__FBSDID("$FreeBSD$"); 27 28#ifdef HAVE_SYS_IOCTL_H 29#include <sys/ioctl.h> 30#endif 31#ifdef HAVE_SYS_PARAM_H 32#include <sys/param.h> 33#endif 34#ifdef HAVE_FCNTL_H 35#include <fcntl.h> 36#endif 37#ifdef HAVE_LIMITS_H 38#include <limits.h> 39#endif 40#ifdef HAVE_UNISTD_H 41#include <unistd.h> 42#endif 43#ifdef HAVE_LINUX_TYPES_H 44#include <linux/types.h> 45#endif 46#ifdef HAVE_LINUX_FIEMAP_H 47#include <linux/fiemap.h> 48#endif 49#ifdef HAVE_LINUX_FS_H 50#include <linux/fs.h> 51#endif 52 53/* The logic to compare sparse file data read from disk with the 54 * specification is a little involved. Set to 1 to have the progress 55 * dumped. */ 56#define DEBUG 0 57 58/* 59 * NOTE: On FreeBSD and Solaris, this test needs ZFS. 60 * You may should perfom this test as 61 * 'TMPDIR=<a directory on the ZFS> libarchive_test'. 62 */ 63 64struct sparse { 65 enum { DATA, HOLE, END } type; 66 size_t size; 67}; 68 69static void create_sparse_file(const char *, const struct sparse *); 70 71#if defined(_WIN32) && !defined(__CYGWIN__) 72#include <winioctl.h> 73/* 74 * Create a sparse file on Windows. 75 */ 76 77#if !defined(PATH_MAX) 78#define PATH_MAX MAX_PATH 79#endif 80#if !defined(__BORLANDC__) 81#define getcwd _getcwd 82#endif 83 84static int 85is_sparse_supported(const char *path) 86{ 87 char root[MAX_PATH+1]; 88 char vol[MAX_PATH+1]; 89 char sys[MAX_PATH+1]; 90 DWORD flags; 91 BOOL r; 92 93 strncpy(root, path, sizeof(root)-1); 94 if (((root[0] >= 'c' && root[0] <= 'z') || 95 (root[0] >= 'C' && root[0] <= 'Z')) && 96 root[1] == ':' && 97 (root[2] == '\\' || root[2] == '/')) 98 root[3] = '\0'; 99 else 100 return (0); 101 assertEqualInt((r = GetVolumeInformation(root, vol, 102 sizeof(vol), NULL, NULL, &flags, sys, sizeof(sys))), 1); 103 return (r != 0 && (flags & FILE_SUPPORTS_SPARSE_FILES) != 0); 104} 105 106static void 107create_sparse_file(const char *path, const struct sparse *s) 108{ 109 char buff[1024]; 110 HANDLE handle; 111 DWORD dmy; 112 113 memset(buff, ' ', sizeof(buff)); 114 115 handle = CreateFileA(path, GENERIC_WRITE, 0, 116 NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, 117 NULL); 118 assert(handle != INVALID_HANDLE_VALUE); 119 assert(DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0, 120 NULL, 0, &dmy, NULL) != 0); 121 while (s->type != END) { 122 if (s->type == HOLE) { 123 LARGE_INTEGER distance; 124 125 distance.QuadPart = s->size; 126 assert(SetFilePointerEx(handle, distance, 127 NULL, FILE_CURRENT) != 0); 128 } else { 129 DWORD w, wr; 130 size_t size; 131 132 size = s->size; 133 while (size) { 134 if (size > sizeof(buff)) 135 w = sizeof(buff); 136 else 137 w = (DWORD)size; 138 assert(WriteFile(handle, buff, w, &wr, NULL) != 0); 139 size -= wr; 140 } 141 } 142 s++; 143 } 144 assertEqualInt(CloseHandle(handle), 1); 145} 146 147#else 148 149#if defined(HAVE_LINUX_FIEMAP_H) 150/* 151 * FIEMAP, which can detect 'hole' of a sparse file, has 152 * been supported from 2.6.28 153 */ 154 155static int 156is_sparse_supported_fiemap(const char *path) 157{ 158 const struct sparse sparse_file[] = { 159 /* This hole size is too small to create a sparse 160 * files for almost filesystem. */ 161 { HOLE, 1024 }, { DATA, 10240 }, 162 { END, 0 } 163 }; 164 int fd, r; 165 struct fiemap *fm; 166 char buff[1024]; 167 const char *testfile = "can_sparse"; 168 169 (void)path; /* UNUSED */ 170 memset(buff, 0, sizeof(buff)); 171 create_sparse_file(testfile, sparse_file); 172 fd = open(testfile, O_RDWR); 173 if (fd < 0) 174 return (0); 175 fm = (struct fiemap *)buff; 176 fm->fm_start = 0; 177 fm->fm_length = ~0ULL;; 178 fm->fm_flags = FIEMAP_FLAG_SYNC; 179 fm->fm_extent_count = (sizeof(buff) - sizeof(*fm))/ 180 sizeof(struct fiemap_extent); 181 r = ioctl(fd, FS_IOC_FIEMAP, fm); 182 close(fd); 183 unlink(testfile); 184 return (r >= 0); 185} 186 187#if !defined(SEEK_HOLE) || !defined(SEEK_DATA) 188static int 189is_sparse_supported(const char *path) 190{ 191 return is_sparse_supported_fiemap(path); 192} 193#endif 194#endif 195 196#if defined(_PC_MIN_HOLE_SIZE) 197 198/* 199 * FreeBSD and Solaris can detect 'hole' of a sparse file 200 * through lseek(HOLE) on ZFS. (UFS does not support yet) 201 */ 202 203static int 204is_sparse_supported(const char *path) 205{ 206 return (pathconf(path, _PC_MIN_HOLE_SIZE) > 0); 207} 208 209#elif defined(SEEK_HOLE) && defined(SEEK_DATA) 210 211static int 212is_sparse_supported(const char *path) 213{ 214 const struct sparse sparse_file[] = { 215 /* This hole size is too small to create a sparse 216 * files for almost filesystem. */ 217 { HOLE, 1024 }, { DATA, 10240 }, 218 { END, 0 } 219 }; 220 int fd, r; 221 const char *testfile = "can_sparse"; 222 223 (void)path; /* UNUSED */ 224 create_sparse_file(testfile, sparse_file); 225 fd = open(testfile, O_RDWR); 226 if (fd < 0) 227 return (0); 228 r = lseek(fd, 0, SEEK_HOLE); 229 close(fd); 230 unlink(testfile); 231#if defined(HAVE_LINUX_FIEMAP_H) 232 if (r < 0) 233 return (is_sparse_supported_fiemap(path)); 234#endif 235 return (r >= 0); 236} 237 238#elif !defined(HAVE_LINUX_FIEMAP_H) 239 240/* 241 * Other system may do not have the API such as lseek(HOLE), 242 * which detect 'hole' of a sparse file. 243 */ 244 245static int 246is_sparse_supported(const char *path) 247{ 248 (void)path; /* UNUSED */ 249 return (0); 250} 251 252#endif 253 254/* 255 * Create a sparse file on POSIX like system. 256 */ 257 258static void 259create_sparse_file(const char *path, const struct sparse *s) 260{ 261 char buff[1024]; 262 int fd; 263 size_t total_size = 0; 264 const struct sparse *cur = s; 265 266 memset(buff, ' ', sizeof(buff)); 267 assert((fd = open(path, O_CREAT | O_WRONLY, 0600)) != -1); 268 269 /* Handle holes at the end by extending the file */ 270 while (cur->type != END) { 271 total_size += cur->size; 272 ++cur; 273 } 274 assert(ftruncate(fd, total_size) != -1); 275 276 while (s->type != END) { 277 if (s->type == HOLE) { 278 assert(lseek(fd, s->size, SEEK_CUR) != (off_t)-1); 279 } else { 280 size_t w, size; 281 282 size = s->size; 283 while (size) { 284 if (size > sizeof(buff)) 285 w = sizeof(buff); 286 else 287 w = size; 288 assert(write(fd, buff, w) != (ssize_t)-1); 289 size -= w; 290 } 291 } 292 s++; 293 } 294 close(fd); 295} 296 297#endif 298 299/* 300 * Sparse test with directory traversals. 301 */ 302static void 303verify_sparse_file(struct archive *a, const char *path, 304 const struct sparse *sparse, int expected_holes) 305{ 306 struct archive_entry *ae; 307 const void *buff; 308 size_t bytes_read; 309 int64_t offset, expected_offset, last_offset; 310 int holes_seen = 0; 311 312 create_sparse_file(path, sparse); 313 assert((ae = archive_entry_new()) != NULL); 314 assertEqualIntA(a, ARCHIVE_OK, archive_read_disk_open(a, path)); 315 assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header2(a, ae)); 316 317 expected_offset = 0; 318 last_offset = 0; 319 while (ARCHIVE_OK == archive_read_data_block(a, &buff, &bytes_read, 320 &offset)) { 321 const char *start = buff; 322#if DEBUG 323 fprintf(stderr, "%s: bytes_read=%d offset=%d\n", path, (int)bytes_read, (int)offset); 324#endif 325 if (offset > last_offset) { 326 ++holes_seen; 327 } 328 /* Blocks entirely before the data we just read. */ 329 while (expected_offset + (int64_t)sparse->size < offset) { 330#if DEBUG 331 fprintf(stderr, " skipping expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 332#endif 333 /* Must be holes. */ 334 assert(sparse->type == HOLE); 335 expected_offset += sparse->size; 336 ++sparse; 337 } 338 /* Block that overlaps beginning of data */ 339 if (expected_offset < offset 340 && expected_offset + (int64_t)sparse->size <= offset + (int64_t)bytes_read) { 341 const char *end = (const char *)buff + (expected_offset - offset) + (size_t)sparse->size; 342#if DEBUG 343 fprintf(stderr, " overlapping hole expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 344#endif 345 /* Must be a hole, overlap must be filled with '\0' */ 346 if (assert(sparse->type == HOLE)) { 347 assertMemoryFilledWith(start, end - start, '\0'); 348 } 349 start = end; 350 expected_offset += sparse->size; 351 ++sparse; 352 } 353 /* Blocks completely contained in data we just read. */ 354 while (expected_offset + (int64_t)sparse->size <= offset + (int64_t)bytes_read) { 355 const char *end = (const char *)buff + (expected_offset - offset) + (size_t)sparse->size; 356 if (sparse->type == HOLE) { 357#if DEBUG 358 fprintf(stderr, " contained hole expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 359#endif 360 361 /* verify data corresponding to hole is '\0' */ 362 if (end > (const char *)buff + bytes_read) { 363 end = (const char *)buff + bytes_read; 364 } 365 assertMemoryFilledWith(start, end - start, '\0'); 366 start = end; 367 expected_offset += sparse->size; 368 ++sparse; 369 } else if (sparse->type == DATA) { 370#if DEBUG 371 fprintf(stderr, " contained data expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 372#endif 373 /* verify data corresponding to hole is ' ' */ 374 if (assert(expected_offset + sparse->size <= offset + bytes_read)) { 375 assert(start == (const char *)buff + (size_t)(expected_offset - offset)); 376 assertMemoryFilledWith(start, end - start, ' '); 377 } 378 start = end; 379 expected_offset += sparse->size; 380 ++sparse; 381 } else { 382 break; 383 } 384 } 385 /* Block that overlaps end of data */ 386 if (expected_offset < offset + (int64_t)bytes_read) { 387 const char *end = (const char *)buff + bytes_read; 388#if DEBUG 389 fprintf(stderr, " trailing overlap expected_offset=%d, size=%d\n", (int)expected_offset, (int)sparse->size); 390#endif 391 /* Must be a hole, overlap must be filled with '\0' */ 392 if (assert(sparse->type == HOLE)) { 393 assertMemoryFilledWith(start, end - start, '\0'); 394 } 395 } 396 last_offset = offset + bytes_read; 397 } 398 /* Count a hole at EOF? */ 399 if (last_offset < archive_entry_size(ae)) { 400 ++holes_seen; 401 } 402 403 /* Verify blocks after last read */ 404 while (sparse->type == HOLE) { 405 expected_offset += sparse->size; 406 ++sparse; 407 } 408 assert(sparse->type == END); 409 assertEqualInt(expected_offset, archive_entry_size(ae)); 410 411 assertEqualInt(holes_seen, expected_holes); 412 413 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); 414 archive_entry_free(ae); 415} 416 417#if defined(_WIN32) && !defined(__CYGWIN__) 418#define close _close 419#define open _open 420#endif 421 422/* 423 * Sparse test without directory traversals. 424 */ 425static void 426verify_sparse_file2(struct archive *a, const char *path, 427 const struct sparse *sparse, int blocks, int preopen) 428{ 429 struct archive_entry *ae; 430 int fd; 431 432 (void)sparse; /* UNUSED */ 433 assert((ae = archive_entry_new()) != NULL); 434 archive_entry_set_pathname(ae, path); 435 if (preopen) 436 fd = open(path, O_RDONLY | O_BINARY); 437 else 438 fd = -1; 439 assertEqualIntA(a, ARCHIVE_OK, 440 archive_read_disk_entry_from_file(a, ae, fd, NULL)); 441 if (fd >= 0) 442 close(fd); 443 /* Verify the number of holes only, not its offset nor its 444 * length because those alignments are deeply dependence on 445 * its filesystem. */ 446 assertEqualInt(blocks, archive_entry_sparse_count(ae)); 447 archive_entry_free(ae); 448} 449 450static void 451test_sparse_whole_file_data() 452{ 453 struct archive_entry *ae; 454 int64_t offset; 455 int i; 456 457 assert((ae = archive_entry_new()) != NULL); 458 archive_entry_set_size(ae, 1024*10); 459 460 /* 461 * Add sparse block data up to the file size. 462 */ 463 offset = 0; 464 for (i = 0; i < 10; i++) { 465 archive_entry_sparse_add_entry(ae, offset, 1024); 466 offset += 1024; 467 } 468 469 failure("There should be no sparse"); 470 assertEqualInt(0, archive_entry_sparse_count(ae)); 471 archive_entry_free(ae); 472} 473 474DEFINE_TEST(test_sparse_basic) 475{ 476 char *cwd; 477 struct archive *a; 478 /* 479 * The alignment of the hole of sparse files deeply depends 480 * on filesystem. In my experience, sparse_file2 test with 481 * 204800 bytes hole size did not pass on ZFS and the result 482 * of that test seemed the size was too small, thus you should 483 * keep a hole size more than 409600 bytes to pass this test 484 * on all platform. 485 */ 486 const struct sparse sparse_file0[] = { 487 { DATA, 1024 }, { HOLE, 2048000 }, 488 { DATA, 2048 }, { HOLE, 2048000 }, 489 { DATA, 4096 }, { HOLE, 20480000 }, 490 { DATA, 8192 }, { HOLE, 204800000 }, 491 { DATA, 1 }, { END, 0 } 492 }; 493 const struct sparse sparse_file1[] = { 494 { HOLE, 409600 }, { DATA, 1 }, 495 { HOLE, 409600 }, { DATA, 1 }, 496 { HOLE, 409600 }, { END, 0 } 497 }; 498 const struct sparse sparse_file2[] = { 499 { HOLE, 409600 * 1 }, { DATA, 1024 }, 500 { HOLE, 409600 * 2 }, { DATA, 1024 }, 501 { HOLE, 409600 * 3 }, { DATA, 1024 }, 502 { HOLE, 409600 * 4 }, { DATA, 1024 }, 503 { HOLE, 409600 * 5 }, { DATA, 1024 }, 504 { HOLE, 409600 * 6 }, { DATA, 1024 }, 505 { HOLE, 409600 * 7 }, { DATA, 1024 }, 506 { HOLE, 409600 * 8 }, { DATA, 1024 }, 507 { HOLE, 409600 * 9 }, { DATA, 1024 }, 508 { HOLE, 409600 * 10}, { DATA, 1024 },/* 10 */ 509 { HOLE, 409600 * 1 }, { DATA, 1024 * 1 }, 510 { HOLE, 409600 * 2 }, { DATA, 1024 * 2 }, 511 { HOLE, 409600 * 3 }, { DATA, 1024 * 3 }, 512 { HOLE, 409600 * 4 }, { DATA, 1024 * 4 }, 513 { HOLE, 409600 * 5 }, { DATA, 1024 * 5 }, 514 { HOLE, 409600 * 6 }, { DATA, 1024 * 6 }, 515 { HOLE, 409600 * 7 }, { DATA, 1024 * 7 }, 516 { HOLE, 409600 * 8 }, { DATA, 1024 * 8 }, 517 { HOLE, 409600 * 9 }, { DATA, 1024 * 9 }, 518 { HOLE, 409600 * 10}, { DATA, 1024 * 10},/* 20 */ 519 { END, 0 } 520 }; 521 const struct sparse sparse_file3[] = { 522 /* This hole size is too small to create a sparse file */ 523 { HOLE, 1 }, { DATA, 10240 }, 524 { HOLE, 1 }, { DATA, 10240 }, 525 { HOLE, 1 }, { DATA, 10240 }, 526 { END, 0 } 527 }; 528 529 /* 530 * Test for the case that sparse data indicates just the whole file 531 * data. 532 */ 533 test_sparse_whole_file_data(); 534 535 /* Check if the filesystem where CWD on can 536 * report the number of the holes of a sparse file. */ 537#ifdef PATH_MAX 538 cwd = getcwd(NULL, PATH_MAX);/* Solaris getcwd needs the size. */ 539#else 540 cwd = getcwd(NULL, 0); 541#endif 542 if (!assert(cwd != NULL)) 543 return; 544 if (!is_sparse_supported(cwd)) { 545 free(cwd); 546 skipping("This filesystem or platform do not support " 547 "the reporting of the holes of a sparse file through " 548 "API such as lseek(HOLE)"); 549 return; 550 } 551 552 /* 553 * Get sparse data through directory traversals. 554 */ 555 assert((a = archive_read_disk_new()) != NULL); 556 557 verify_sparse_file(a, "file0", sparse_file0, 4); 558 verify_sparse_file(a, "file1", sparse_file1, 3); 559 verify_sparse_file(a, "file2", sparse_file2, 20); 560 /* Encoded non sparse; expect a data block but no sparse entries. */ 561 verify_sparse_file(a, "file3", sparse_file3, 0); 562 563 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 564 565 /* 566 * Get sparse data through archive_read_disk_entry_from_file(). 567 */ 568 assert((a = archive_read_disk_new()) != NULL); 569 570 verify_sparse_file2(a, "file0", sparse_file0, 5, 0); 571 verify_sparse_file2(a, "file0", sparse_file0, 5, 1); 572 573 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 574 free(cwd); 575} 576 577DEFINE_TEST(test_fully_sparse_files) 578{ 579 char *cwd; 580 struct archive *a; 581 582 const struct sparse sparse_file[] = { 583 { HOLE, 409600 }, { END, 0 } 584 }; 585 /* Check if the filesystem where CWD on can 586 * report the number of the holes of a sparse file. */ 587#ifdef PATH_MAX 588 cwd = getcwd(NULL, PATH_MAX);/* Solaris getcwd needs the size. */ 589#else 590 cwd = getcwd(NULL, 0); 591#endif 592 if (!assert(cwd != NULL)) 593 return; 594 if (!is_sparse_supported(cwd)) { 595 free(cwd); 596 skipping("This filesystem or platform do not support " 597 "the reporting of the holes of a sparse file through " 598 "API such as lseek(HOLE)"); 599 return; 600 } 601 602 assert((a = archive_read_disk_new()) != NULL); 603 604 /* Fully sparse files are encoded with a zero-length "data" block. */ 605 verify_sparse_file(a, "file0", sparse_file, 1); 606 607 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 608 free(cwd); 609} 610