test_write_format_zip_large.c revision 318482
1/*- 2 * Copyright (c) 2003-2007,2013 Tim Kientzle 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25#include "test.h" 26__FBSDID("$FreeBSD: stable/11/contrib/libarchive/libarchive/test/test_write_format_zip_large.c 318482 2017-05-18 19:47:43Z mm $"); 27 28#include <errno.h> 29#include <stdlib.h> 30#include <string.h> 31 32/* 33 * This is a somewhat tricky test that verifies the ability to 34 * write and read very large entries to zip archives. 35 * 36 * See test_tar_large.c for more information about the machinery 37 * being used here. 38 */ 39 40static size_t nullsize; 41static void *nulldata; 42 43struct fileblock { 44 struct fileblock *next; 45 int size; 46 void *buff; 47 int64_t gap_size; /* Size of following gap */ 48}; 49 50struct fileblocks { 51 int64_t filesize; 52 int64_t fileposition; 53 int64_t gap_remaining; 54 void *buff; 55 struct fileblock *first; 56 struct fileblock *current; 57 struct fileblock *last; 58}; 59 60/* The following size definitions simplify things below. */ 61#define KB ((int64_t)1024) 62#define MB ((int64_t)1024 * KB) 63#define GB ((int64_t)1024 * MB) 64#define TB ((int64_t)1024 * GB) 65 66static int64_t memory_read_skip(struct archive *, void *, int64_t request); 67static ssize_t memory_read(struct archive *, void *, const void **buff); 68static ssize_t memory_write(struct archive *, void *, const void *, size_t); 69 70static uint16_t le16(const void *_p) { 71 const uint8_t *p = _p; 72 return p[0] | (p[1] << 8); 73} 74 75static uint32_t le32(const void *_p) { 76 const uint8_t *p = _p; 77 return le16(p) | ((uint32_t)le16(p + 2) << 16); 78} 79 80static uint64_t le64(const void *_p) { 81 const uint8_t *p = _p; 82 return le32(p) | ((uint64_t)le32(p + 4) << 32); 83} 84 85static ssize_t 86memory_write(struct archive *a, void *_private, const void *buff, size_t size) 87{ 88 struct fileblocks *private = _private; 89 struct fileblock *block; 90 91 (void)a; 92 93 if ((const char *)nulldata <= (const char *)buff 94 && (const char *)buff < (const char *)nulldata + nullsize) { 95 /* We don't need to store a block of gap data. */ 96 private->last->gap_size += (int64_t)size; 97 } else { 98 /* Yes, we're assuming the very first write is metadata. */ 99 /* It's header or metadata, copy and save it. */ 100 block = (struct fileblock *)malloc(sizeof(*block)); 101 memset(block, 0, sizeof(*block)); 102 block->size = (int)size; 103 block->buff = malloc(size); 104 memcpy(block->buff, buff, size); 105 if (private->last == NULL) { 106 private->first = private->last = block; 107 } else { 108 private->last->next = block; 109 private->last = block; 110 } 111 block->next = NULL; 112 } 113 private->filesize += size; 114 return ((long)size); 115} 116 117static ssize_t 118memory_read(struct archive *a, void *_private, const void **buff) 119{ 120 struct fileblocks *private = _private; 121 ssize_t size; 122 123 (void)a; 124 125 while (private->current != NULL && private->buff == NULL && private->gap_remaining == 0) { 126 private->current = private->current->next; 127 if (private->current != NULL) { 128 private->buff = private->current->buff; 129 private->gap_remaining = private->current->gap_size; 130 } 131 } 132 133 if (private->current == NULL) 134 return (0); 135 136 /* If there's real data, return that. */ 137 if (private->buff != NULL) { 138 *buff = private->buff; 139 size = ((char *)private->current->buff + private->current->size) 140 - (char *)private->buff; 141 private->buff = NULL; 142 private->fileposition += size; 143 return (size); 144 } 145 146 /* Big gap: too big to return all at once, so just return some. */ 147 if (private->gap_remaining > (int64_t)nullsize) { 148 private->gap_remaining -= nullsize; 149 *buff = nulldata; 150 private->fileposition += nullsize; 151 return (nullsize); 152 } 153 154 /* Small gap: finish the gap and prep for next block. */ 155 if (private->gap_remaining > 0) { 156 size = (ssize_t)private->gap_remaining; 157 *buff = nulldata; 158 private->gap_remaining = 0; 159 private->fileposition += size; 160 161 private->current = private->current->next; 162 if (private->current != NULL) { 163 private->buff = private->current->buff; 164 private->gap_remaining = private->current->gap_size; 165 } 166 167 return (size); 168 } 169 fprintf(stderr, "\n\n\nInternal failure\n\n\n"); 170 exit(1); 171} 172 173static int 174memory_read_open(struct archive *a, void *_private) 175{ 176 struct fileblocks *private = _private; 177 178 (void)a; /* UNUSED */ 179 180 private->current = private->first; 181 private->fileposition = 0; 182 if (private->current != NULL) { 183 private->buff = private->current->buff; 184 private->gap_remaining = private->current->gap_size; 185 } 186 return (ARCHIVE_OK); 187} 188 189static int64_t 190memory_read_seek(struct archive *a, void *_private, int64_t offset, int whence) 191{ 192 struct fileblocks *private = _private; 193 194 (void)a; 195 if (whence == SEEK_END) { 196 offset = private->filesize + offset; 197 } else if (whence == SEEK_CUR) { 198 offset = private->fileposition + offset; 199 } 200 201 if (offset < 0) { 202 fprintf(stderr, "\n\n\nInternal failure: negative seek\n\n\n"); 203 exit(1); 204 } 205 206 /* We've converted the request into a SEEK_SET. */ 207 private->fileposition = offset; 208 209 /* Walk the block list to find the new position. */ 210 offset = 0; 211 private->current = private->first; 212 while (private->current != NULL) { 213 if (offset + private->current->size > private->fileposition) { 214 /* Position is in this block. */ 215 private->buff = (char *)private->current->buff 216 + private->fileposition - offset; 217 private->gap_remaining = private->current->gap_size; 218 return private->fileposition; 219 } 220 offset += private->current->size; 221 if (offset + private->current->gap_size > private->fileposition) { 222 /* Position is in this gap. */ 223 private->buff = NULL; 224 private->gap_remaining = private->current->gap_size 225 - (private->fileposition - offset); 226 return private->fileposition; 227 } 228 offset += private->current->gap_size; 229 /* Skip to next block. */ 230 private->current = private->current->next; 231 } 232 if (private->fileposition == private->filesize) { 233 return private->fileposition; 234 } 235 fprintf(stderr, "\n\n\nInternal failure: over-sized seek\n\n\n"); 236 exit(1); 237} 238 239static int64_t 240memory_read_skip(struct archive *a, void *_private, int64_t skip) 241{ 242 struct fileblocks *private = _private; 243 int64_t old_position = private->fileposition; 244 int64_t new_position = memory_read_seek(a, _private, skip, SEEK_CUR); 245 return (new_position - old_position); 246} 247 248static struct fileblocks * 249fileblocks_new(void) 250{ 251 struct fileblocks *fileblocks; 252 253 fileblocks = calloc(1, sizeof(struct fileblocks)); 254 return fileblocks; 255} 256 257static void 258fileblocks_free(struct fileblocks *fileblocks) 259{ 260 while (fileblocks->first != NULL) { 261 struct fileblock *b = fileblocks->first; 262 fileblocks->first = fileblocks->first->next; 263 free(b->buff); 264 free(b); 265 } 266 free(fileblocks); 267} 268 269 270/* The sizes of the entries we're going to generate. */ 271static int64_t test_sizes[] = { 272 /* Test for 32-bit signed overflow. */ 273 2 * GB - 1, 2 * GB, 2 * GB + 1, 274 /* Test for 32-bit unsigned overflow. */ 275 4 * GB - 1, 4 * GB, 4 * GB + 1, 276 /* And beyond ... because we can. */ 277 16 * GB - 1, 16 * GB, 16 * GB + 1, 278 64 * GB - 1, 64 * GB, 64 * GB + 1, 279 256 * GB - 1, 256 * GB, 256 * GB + 1, 280 1 * TB, 281 0 282}; 283 284 285static void 286verify_large_zip(struct archive *a, struct fileblocks *fileblocks) 287{ 288 char namebuff[64]; 289 struct archive_entry *ae; 290 int i; 291 292 assertEqualIntA(a, ARCHIVE_OK, 293 archive_read_set_options(a, "zip:ignorecrc32")); 294 assertEqualIntA(a, ARCHIVE_OK, 295 archive_read_set_open_callback(a, memory_read_open)); 296 assertEqualIntA(a, ARCHIVE_OK, 297 archive_read_set_read_callback(a, memory_read)); 298 assertEqualIntA(a, ARCHIVE_OK, 299 archive_read_set_skip_callback(a, memory_read_skip)); 300 assertEqualIntA(a, ARCHIVE_OK, 301 archive_read_set_seek_callback(a, memory_read_seek)); 302 assertEqualIntA(a, ARCHIVE_OK, 303 archive_read_set_callback_data(a, fileblocks)); 304 assertEqualIntA(a, ARCHIVE_OK, archive_read_open1(a)); 305 306 /* 307 * Read entries back. 308 */ 309 for (i = 0; test_sizes[i] > 0; i++) { 310 assertEqualIntA(a, ARCHIVE_OK, 311 archive_read_next_header(a, &ae)); 312 sprintf(namebuff, "file_%d", i); 313 assertEqualString(namebuff, archive_entry_pathname(ae)); 314 assertEqualInt(test_sizes[i], archive_entry_size(ae)); 315 } 316 assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 317 assertEqualString("lastfile", archive_entry_pathname(ae)); 318 319 assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); 320 321 /* Close out the archive. */ 322 assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); 323} 324 325DEFINE_TEST(test_write_format_zip_large) 326{ 327 int i; 328 char namebuff[64]; 329 struct fileblocks *fileblocks = fileblocks_new(); 330 struct archive_entry *ae; 331 struct archive *a; 332 const char *p; 333 const char *cd_start, *zip64_eocd, *zip64_locator, *eocd; 334 int64_t cd_size; 335 char *buff; 336 int64_t filesize; 337 size_t writesize, buffsize, s; 338 339 nullsize = (size_t)(1 * MB); 340 nulldata = malloc(nullsize); 341 memset(nulldata, 0xAA, nullsize); 342 343 /* 344 * Open an archive for writing. 345 */ 346 a = archive_write_new(); 347 archive_write_set_format_zip(a); 348 assertEqualIntA(a, ARCHIVE_OK, 349 archive_write_set_options(a, "zip:compression=store")); 350 assertEqualIntA(a, ARCHIVE_OK, 351 archive_write_set_options(a, "zip:fakecrc32")); 352 assertEqualIntA(a, ARCHIVE_OK, 353 archive_write_set_bytes_per_block(a, 0)); /* No buffering. */ 354 assertEqualIntA(a, ARCHIVE_OK, 355 archive_write_open(a, fileblocks, NULL, memory_write, NULL)); 356 357 /* 358 * Write a series of large files to it. 359 */ 360 for (i = 0; test_sizes[i] != 0; i++) { 361 assert((ae = archive_entry_new()) != NULL); 362 sprintf(namebuff, "file_%d", i); 363 archive_entry_copy_pathname(ae, namebuff); 364 archive_entry_set_mode(ae, S_IFREG | 0755); 365 filesize = test_sizes[i]; 366 archive_entry_set_size(ae, filesize); 367 368 assertEqualIntA(a, ARCHIVE_OK, 369 archive_write_header(a, ae)); 370 archive_entry_free(ae); 371 372 /* 373 * Write the actual data to the archive. 374 */ 375 while (filesize > 0) { 376 writesize = nullsize; 377 if ((int64_t)writesize > filesize) 378 writesize = (size_t)filesize; 379 assertEqualIntA(a, (int)writesize, 380 (int)archive_write_data(a, nulldata, writesize)); 381 filesize -= writesize; 382 } 383 } 384 385 assert((ae = archive_entry_new()) != NULL); 386 archive_entry_copy_pathname(ae, "lastfile"); 387 archive_entry_set_mode(ae, S_IFREG | 0755); 388 assertA(0 == archive_write_header(a, ae)); 389 archive_entry_free(ae); 390 391 /* Close out the archive. */ 392 assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); 393 assertEqualInt(ARCHIVE_OK, archive_write_free(a)); 394 395 /* 396 * Read back with seeking reader: 397 */ 398 a = archive_read_new(); 399 assertEqualIntA(a, ARCHIVE_OK, 400 archive_read_support_format_zip_seekable(a)); 401 verify_large_zip(a, fileblocks); 402 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 403 404 /* 405 * Read back with streaming reader: 406 */ 407 a = archive_read_new(); 408 assertEqualIntA(a, ARCHIVE_OK, 409 archive_read_support_format_zip_streamable(a)); 410 verify_large_zip(a, fileblocks); 411 assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 412 413 /* 414 * Manually verify some of the final bytes of the archives. 415 */ 416 /* Collect the final bytes together */ 417#define FINAL_SIZE 8192 418 buff = malloc(FINAL_SIZE); 419 buffsize = 0; 420 memory_read_open(NULL, fileblocks); 421 memory_read_seek(NULL, fileblocks, -FINAL_SIZE, SEEK_END); 422 while ((s = memory_read(NULL, fileblocks, (const void **)&p)) > 0) { 423 memcpy(buff + buffsize, p, s); 424 buffsize += s; 425 } 426 assertEqualInt(buffsize, FINAL_SIZE); 427 428 p = buff + buffsize; 429 430 /* Verify regular end-of-central-directory record */ 431 eocd = p - 22; 432 assertEqualMem(eocd, "PK\005\006\0\0\0\0", 8); 433 assertEqualMem(eocd + 8, "\021\0\021\0", 4); /* 17 entries total */ 434 cd_size = le32(eocd + 12); 435 /* Start of CD offset should be 0xffffffff */ 436 assertEqualMem(eocd + 16, "\xff\xff\xff\xff", 4); 437 assertEqualMem(eocd + 20, "\0\0", 2); /* No Zip comment */ 438 439 /* Verify Zip64 locator */ 440 zip64_locator = p - 42; 441 assertEqualMem(zip64_locator, "PK\006\007\0\0\0\0", 8); 442 zip64_eocd = p - (fileblocks->filesize - le64(zip64_locator + 8)); 443 assertEqualMem(zip64_locator + 16, "\001\0\0\0", 4); 444 445 /* Verify Zip64 end-of-cd record. */ 446 assert(zip64_eocd == p - 98); 447 assertEqualMem(zip64_eocd, "PK\006\006", 4); 448 assertEqualInt(44, le64(zip64_eocd + 4)); // Size of EoCD record - 12 449 assertEqualMem(zip64_eocd + 12, "\055\0", 2); // Made by version: 45 450 assertEqualMem(zip64_eocd + 14, "\055\0", 2); // Requires version: 45 451 assertEqualMem(zip64_eocd + 16, "\0\0\0\0", 4); // This disk 452 assertEqualMem(zip64_eocd + 20, "\0\0\0\0", 4); // Total disks 453 assertEqualInt(17, le64(zip64_eocd + 24)); // Entries on this disk 454 assertEqualInt(17, le64(zip64_eocd + 32)); // Total entries 455 cd_size = le64(zip64_eocd + 40); 456 cd_start = p - (fileblocks->filesize - le64(zip64_eocd + 48)); 457 458 assert(cd_start + cd_size == zip64_eocd); 459 460 assertEqualInt(le64(zip64_eocd + 48) // Start of CD 461 + cd_size 462 + 56 // Size of Zip64 EOCD 463 + 20 // Size of Zip64 locator 464 + 22, // Size of EOCD 465 fileblocks->filesize); 466 467 // TODO: Scan entire Central Directory, sanity-check all data 468 assertEqualMem(cd_start, "PK\001\002", 4); 469 470 fileblocks_free(fileblocks); 471 free(buff); 472 free(nulldata); 473} 474