test_write_format_zip_large.c revision 313570
1/*- 2 * Copyright (c) 2003-2007,2013 Tim Kientzle 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25#include "test.h" 26__FBSDID("$FreeBSD: stable/11/contrib/libarchive/libarchive/test/test_write_format_zip_large.c 313570 2017-02-11 00:54:16Z mm $"); 27 28#include <errno.h> 29#include <stdlib.h> 30#include <string.h> 31 32/* 33 * This is a somewhat tricky test that verifies the ability to 34 * write and read very large entries to zip archives. 35 * 36 * See test_tar_large.c for more information about the machinery 37 * being used here. 38 */ 39 40static size_t nullsize; 41static void *nulldata; 42 43struct fileblock { 44 struct fileblock *next; 45 int size; 46 void *buff; 47 int64_t gap_size; /* Size of following gap */ 48}; 49 50struct fileblocks { 51 int64_t filesize; 52 int64_t fileposition; 53 int64_t gap_remaining; 54 void *buff; 55 struct fileblock *first; 56 struct fileblock *current; 57 struct fileblock *last; 58}; 59 60/* The following size definitions simplify things below. */ 61#define KB ((int64_t)1024) 62#define MB ((int64_t)1024 * KB) 63#define GB ((int64_t)1024 * MB) 64#define TB ((int64_t)1024 * GB) 65 66static int64_t memory_read_skip(struct archive *, void *, int64_t request); 67static ssize_t memory_read(struct archive *, void *, const void **buff); 68static ssize_t memory_write(struct archive *, void *, const void *, size_t); 69 70static int16_t le16(const void *_p) { 71 const uint8_t *p = _p; 72 return (0xff & (int16_t)p[0]) | ((0xff & (int16_t)p[1]) << 8); 73} 74 75static int32_t le32(const void *_p) { 76 const uint8_t *p = _p; 77 int32_t v = 0xffff & (int32_t)le16(_p); 78 return v + ((0xffff & (int32_t)le16(p + 2)) << 16); 79} 80 81static int64_t le64(const void *_p) { 82 const uint8_t *p = _p; 83 int64_t v = 0xffffffff & (int64_t)le32(_p); 84 return v + ((0xffffffff & (int64_t)le32(p + 4)) << 32); 85} 86 87static ssize_t 88memory_write(struct archive *a, void *_private, const void *buff, size_t size) 89{ 90 struct fileblocks *private = _private; 91 struct fileblock *block; 92 93 (void)a; 94 95 if ((const char *)nulldata <= (const char *)buff 96 && (const char *)buff < (const char *)nulldata + nullsize) { 97 /* We don't need to store a block of gap data. */ 98 private->last->gap_size += (int64_t)size; 99 } else { 100 /* Yes, we're assuming the very first write is metadata. */ 101 /* It's header or metadata, copy and save it. */ 102 block = (struct fileblock *)malloc(sizeof(*block)); 103 memset(block, 0, sizeof(*block)); 104 block->size = (int)size; 105 block->buff = malloc(size); 106 memcpy(block->buff, buff, size); 107 if (private->last == NULL) { 108 private->first = private->last = block; 109 } else { 110 private->last->next = block; 111 private->last = block; 112 } 113 block->next = NULL; 114 } 115 private->filesize += size; 116 return ((long)size); 117} 118 119static ssize_t 120memory_read(struct archive *a, void *_private, const void **buff) 121{ 122 struct fileblocks *private = _private; 123 ssize_t size; 124 125 (void)a; 126 127 while (private->current != NULL && private->buff == NULL && private->gap_remaining == 0) { 128 private->current = private->current->next; 129 if (private->current != NULL) { 130 private->buff = private->current->buff; 131 private->gap_remaining = private->current->gap_size; 132 } 133 } 134 135 if (private->current == NULL) 136 return (0); 137 138 /* If there's real data, return that. */ 139 if (private->buff != NULL) { 140 *buff = private->buff; 141 size = ((char *)private->current->buff + private->current->size) 142 - (char *)private->buff; 143 private->buff = NULL; 144 private->fileposition += size; 145 return (size); 146 } 147 148 /* Big gap: too big to return all at once, so just return some. */ 149 if (private->gap_remaining > (int64_t)nullsize) { 150 private->gap_remaining -= nullsize; 151 *buff = nulldata; 152 private->fileposition += nullsize; 153 return (nullsize); 154 } 155 156 /* Small gap: finish the gap and prep for next block. */ 157 if (private->gap_remaining > 0) { 158 size = (ssize_t)private->gap_remaining; 159 *buff = nulldata; 160 private->gap_remaining = 0; 161 private->fileposition += size; 162 163 private->current = private->current->next; 164 if (private->current != NULL) { 165 private->buff = private->current->buff; 166 private->gap_remaining = private->current->gap_size; 167 } 168 169 return (size); 170 } 171 fprintf(stderr, "\n\n\nInternal failure\n\n\n"); 172 exit(1); 173} 174 175static int 176memory_read_open(struct archive *a, void *_private) 177{ 178 struct fileblocks *private = _private; 179 180 (void)a; /* UNUSED */ 181 182 private->current = private->first; 183 private->fileposition = 0; 184 if (private->current != NULL) { 185 private->buff = private->current->buff; 186 private->gap_remaining = private->current->gap_size; 187 } 188 return (ARCHIVE_OK); 189} 190 191static int64_t 192memory_read_seek(struct archive *a, void *_private, int64_t offset, int whence) 193{ 194 struct fileblocks *private = _private; 195 196 (void)a; 197 if (whence == SEEK_END) { 198 offset = private->filesize + offset; 199 } else if (whence == SEEK_CUR) { 200 offset = private->fileposition + offset; 201 } 202 203 if (offset < 0) { 204 fprintf(stderr, "\n\n\nInternal failure: negative seek\n\n\n"); 205 exit(1); 206 } 207 208 /* We've converted the request into a SEEK_SET. */ 209 private->fileposition = offset; 210 211 /* Walk the block list to find the new position. */ 212 offset = 0; 213 private->current = private->first; 214 while (private->current != NULL) { 215 if (offset + private->current->size > private->fileposition) { 216 /* Position is in this block. */ 217 private->buff = (char *)private->current->buff 218 + private->fileposition - offset; 219 private->gap_remaining = private->current->gap_size; 220 return private->fileposition; 221 } 222 offset += private->current->size; 223 if (offset + private->current->gap_size > private->fileposition) { 224 /* Position is in this gap. */ 225 private->buff = NULL; 226 private->gap_remaining = private->current->gap_size 227 - (private->fileposition - offset); 228 return private->fileposition; 229 } 230 offset += private->current->gap_size; 231 /* Skip to next block. */ 232 private->current = private->current->next; 233 } 234 if (private->fileposition == private->filesize) { 235 return private->fileposition; 236 } 237 fprintf(stderr, "\n\n\nInternal failure: over-sized seek\n\n\n"); 238 exit(1); 239} 240 241static int64_t 242memory_read_skip(struct archive *a, void *_private, int64_t skip) 243{ 244 struct fileblocks *private = _private; 245 int64_t old_position = private->fileposition; 246 int64_t new_position = memory_read_seek(a, _private, skip, SEEK_CUR); 247 return (new_position - old_position); 248} 249 250static struct fileblocks * 251fileblocks_new(void) 252{ 253 struct fileblocks *fileblocks; 254 255 fileblocks = calloc(1, sizeof(struct fileblocks)); 256 return fileblocks; 257} 258 259static void 260fileblocks_free(struct fileblocks *fileblocks) 261{ 262 while (fileblocks->first != NULL) { 263 struct fileblock *b = fileblocks->first; 264 fileblocks->first = fileblocks->first->next; 265 free(b->buff); 266 free(b); 267 } 268 free(fileblocks); 269} 270 271 272/* The sizes of the entries we're going to generate. */ 273static int64_t test_sizes[] = { 274 /* Test for 32-bit signed overflow. */ 275 2 * GB - 1, 2 * GB, 2 * GB + 1, 276 /* Test for 32-bit unsigned overflow. */ 277 4 * GB - 1, 4 * GB, 4 * GB + 1, 278 /* And beyond ... because we can. */ 279 16 * GB - 1, 16 * GB, 16 * GB + 1, 280 64 * GB - 1, 64 * GB, 64 * GB + 1, 281 256 * GB - 1, 256 * GB, 256 * GB + 1, 282 1 * TB, 283 0 284}; 285 286 287static void 288verify_large_zip(struct archive *a, struct fileblocks *fileblocks) 289{ 290 char namebuff[64]; 291 struct archive_entry *ae; 292 int i; 293 294 assertEqualIntA(a, ARCHIVE_OK, 295 archive_read_set_options(a, "zip:ignorecrc32")); 296 assertEqualIntA(a, ARCHIVE_OK, 297 archive_read_set_open_callback(a, memory_read_open)); 298 assertEqualIntA(a, ARCHIVE_OK, 299 archive_read_set_read_callback(a, memory_read)); 300 assertEqualIntA(a, ARCHIVE_OK, 301 archive_read_set_skip_callback(a, memory_read_skip)); 302 assertEqualIntA(a, ARCHIVE_OK, 303 archive_read_set_seek_callback(a, memory_read_seek)); 304 assertEqualIntA(a, ARCHIVE_OK, 305 archive_read_set_callback_data(a, fileblocks)); 306 assertEqualIntA(a, ARCHIVE_OK, archive_read_open1(a)); 307 308 /* 309 * Read entries back. 310 */ 311 for (i = 0; test_sizes[i] > 0; i++) { 312 assertEqualIntA(a, ARCHIVE_OK, 313 archive_read_next_header(a, &ae)); 314 sprintf(namebuff, "file_%d", i); 315 assertEqualString(namebuff, archive_entry_pathname(ae)); 316 assertEqualInt(test_sizes[i], archive_entry_size(ae)); 317 } 318 assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 319 assertEqualString("lastfile", archive_entry_pathname(ae)); 320 321 assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); 322 323 /* Close out the archive. */ 324 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); 325} 326 327DEFINE_TEST(test_write_format_zip_large) 328{ 329 int i; 330 char namebuff[64]; 331 struct fileblocks *fileblocks = fileblocks_new(); 332 struct archive_entry *ae; 333 struct archive *a; 334 const char *p; 335 const char *cd_start, *zip64_eocd, *zip64_locator, *eocd; 336 int64_t cd_size; 337 char *buff; 338 int64_t filesize; 339 size_t writesize, buffsize, s; 340 341 nullsize = (size_t)(1 * MB); 342 nulldata = malloc(nullsize); 343 memset(nulldata, 0xAA, nullsize); 344 345 /* 346 * Open an archive for writing. 347 */ 348 a = archive_write_new(); 349 archive_write_set_format_zip(a); 350 assertEqualIntA(a, ARCHIVE_OK, 351 archive_write_set_options(a, "zip:compression=store")); 352 assertEqualIntA(a, ARCHIVE_OK, 353 archive_write_set_options(a, "zip:fakecrc32")); 354 assertEqualIntA(a, ARCHIVE_OK, 355 archive_write_set_bytes_per_block(a, 0)); /* No buffering. */ 356 assertEqualIntA(a, ARCHIVE_OK, 357 archive_write_open(a, fileblocks, NULL, memory_write, NULL)); 358 359 /* 360 * Write a series of large files to it. 361 */ 362 for (i = 0; test_sizes[i] != 0; i++) { 363 assert((ae = archive_entry_new()) != NULL); 364 sprintf(namebuff, "file_%d", i); 365 archive_entry_copy_pathname(ae, namebuff); 366 archive_entry_set_mode(ae, S_IFREG | 0755); 367 filesize = test_sizes[i]; 368 archive_entry_set_size(ae, filesize); 369 370 assertEqualIntA(a, ARCHIVE_OK, 371 archive_write_header(a, ae)); 372 archive_entry_free(ae); 373 374 /* 375 * Write the actual data to the archive. 376 */ 377 while (filesize > 0) { 378 writesize = nullsize; 379 if ((int64_t)writesize > filesize) 380 writesize = (size_t)filesize; 381 assertEqualIntA(a, (int)writesize, 382 (int)archive_write_data(a, nulldata, writesize)); 383 filesize -= writesize; 384 } 385 } 386 387 assert((ae = archive_entry_new()) != NULL); 388 archive_entry_copy_pathname(ae, "lastfile"); 389 archive_entry_set_mode(ae, S_IFREG | 0755); 390 assertA(0 == archive_write_header(a, ae)); 391 archive_entry_free(ae); 392 393 /* Close out the archive. */ 394 assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); 395 assertEqualInt(ARCHIVE_OK, archive_write_free(a)); 396 397 /* 398 * Read back with seeking reader: 399 */ 400 a = archive_read_new(); 401 assertEqualIntA(a, ARCHIVE_OK, 402 archive_read_support_format_zip_seekable(a)); 403 verify_large_zip(a, fileblocks); 404 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 405 406 /* 407 * Read back with streaming reader: 408 */ 409 a = archive_read_new(); 410 assertEqualIntA(a, ARCHIVE_OK, 411 archive_read_support_format_zip_streamable(a)); 412 verify_large_zip(a, fileblocks); 413 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 414 415 /* 416 * Manually verify some of the final bytes of the archives. 417 */ 418 /* Collect the final bytes together */ 419#define FINAL_SIZE 8192 420 buff = malloc(FINAL_SIZE); 421 buffsize = 0; 422 memory_read_open(NULL, fileblocks); 423 memory_read_seek(NULL, fileblocks, -FINAL_SIZE, SEEK_END); 424 while ((s = memory_read(NULL, fileblocks, (const void **)&p)) > 0) { 425 memcpy(buff + buffsize, p, s); 426 buffsize += s; 427 } 428 assertEqualInt(buffsize, FINAL_SIZE); 429 430 p = buff + buffsize; 431 432 /* Verify regular end-of-central-directory record */ 433 eocd = p - 22; 434 assertEqualMem(eocd, "PK\005\006\0\0\0\0", 8); 435 assertEqualMem(eocd + 8, "\021\0\021\0", 4); /* 17 entries total */ 436 cd_size = le32(eocd + 12); 437 /* Start of CD offset should be 0xffffffff */ 438 assertEqualMem(eocd + 16, "\xff\xff\xff\xff", 4); 439 assertEqualMem(eocd + 20, "\0\0", 2); /* No Zip comment */ 440 441 /* Verify Zip64 locator */ 442 zip64_locator = p - 42; 443 assertEqualMem(zip64_locator, "PK\006\007\0\0\0\0", 8); 444 zip64_eocd = p - (fileblocks->filesize - le64(zip64_locator + 8)); 445 assertEqualMem(zip64_locator + 16, "\001\0\0\0", 4); 446 447 /* Verify Zip64 end-of-cd record. */ 448 assert(zip64_eocd == p - 98); 449 assertEqualMem(zip64_eocd, "PK\006\006", 4); 450 assertEqualInt(44, le64(zip64_eocd + 4)); // Size of EoCD record - 12 451 assertEqualMem(zip64_eocd + 12, "\055\0", 2); // Made by version: 45 452 assertEqualMem(zip64_eocd + 14, "\055\0", 2); // Requires version: 45 453 assertEqualMem(zip64_eocd + 16, "\0\0\0\0", 4); // This disk 454 assertEqualMem(zip64_eocd + 20, "\0\0\0\0", 4); // Total disks 455 assertEqualInt(17, le64(zip64_eocd + 24)); // Entries on this disk 456 assertEqualInt(17, le64(zip64_eocd + 32)); // Total entries 457 cd_size = le64(zip64_eocd + 40); 458 cd_start = p - (fileblocks->filesize - le64(zip64_eocd + 48)); 459 460 assert(cd_start + cd_size == zip64_eocd); 461 462 assertEqualInt(le64(zip64_eocd + 48) // Start of CD 463 + cd_size 464 + 56 // Size of Zip64 EOCD 465 + 20 // Size of Zip64 locator 466 + 22, // Size of EOCD 467 fileblocks->filesize); 468 469 // TODO: Scan entire Central Directory, sanity-check all data 470 assertEqualMem(cd_start, "PK\001\002", 4); 471 472 fileblocks_free(fileblocks); 473 free(buff); 474 free(nulldata); 475} 476