1228753Smm/*- 2228753Smm * Copyright (c) 2003-2007 Tim Kientzle 3228753Smm * All rights reserved. 4228753Smm * 5228753Smm * Redistribution and use in source and binary forms, with or without 6228753Smm * modification, are permitted provided that the following conditions 7228753Smm * are met: 8228753Smm * 1. Redistributions of source code must retain the above copyright 9228753Smm * notice, this list of conditions and the following disclaimer. 10228753Smm * 2. Redistributions in binary form must reproduce the above copyright 11228753Smm * notice, this list of conditions and the following disclaimer in the 12228753Smm * documentation and/or other materials provided with the distribution. 13228753Smm * 14228753Smm * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15228753Smm * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16228753Smm * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17228753Smm * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18228753Smm * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19228753Smm * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20228753Smm * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21228753Smm * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22228753Smm * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23228753Smm * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24228753Smm */ 25228753Smm#include "test.h" 26229592Smm__FBSDID("$FreeBSD$"); 27228753Smm 28228753Smm#include <errno.h> 29228753Smm#include <stdlib.h> 30228753Smm#include <string.h> 31228753Smm 32228753Smm/* 33228753Smm * This is a somewhat tricky test that verifies the ability to 34228753Smm * write and read very large entries to tar archives. It 35228753Smm * writes entries from 2GB up to 1TB to an archive in memory. 36228753Smm * The memory storage here carefully avoids actually storing 37228753Smm * any part of the file bodies, so it runs very quickly and requires 38228753Smm * very little memory. If you're willing to wait a few minutes, 39228753Smm * you should be able to exercise petabyte entries with this code. 40228753Smm */ 41228753Smm 42228753Smm/* 43228753Smm * Each file is built up by duplicating the following block. 44228753Smm */ 45228753Smmstatic size_t filedatasize; 46228753Smmstatic void *filedata; 47228753Smm 48228753Smm/* 49228753Smm * We store the archive as blocks of data generated by libarchive, 50228753Smm * each possibly followed by bytes of file data. 51228753Smm */ 52228753Smmstruct memblock { 53228753Smm struct memblock *next; 54228753Smm size_t size; 55228753Smm void *buff; 56228753Smm int64_t filebytes; 57228753Smm}; 58228753Smm 59228753Smm/* 60228753Smm * The total memory store is just a list of memblocks plus 61228753Smm * some accounting overhead. 62228753Smm */ 63228753Smmstruct memdata { 64228753Smm int64_t filebytes; 65228753Smm void *buff; 66228753Smm struct memblock *first; 67228753Smm struct memblock *last; 68228753Smm}; 69228753Smm 70228753Smm/* The following size definitions simplify things below. */ 71228753Smm#define KB ((int64_t)1024) 72228753Smm#define MB ((int64_t)1024 * KB) 73228753Smm#define GB ((int64_t)1024 * MB) 74228753Smm#define TB ((int64_t)1024 * GB) 75228753Smm 76228753Smm#if ARCHIVE_VERSION_NUMBER < 2000000 77228753Smmstatic ssize_t memory_read_skip(struct archive *, void *, size_t request); 78228753Smm#else 79228753Smmstatic off_t memory_read_skip(struct archive *, void *, off_t request); 80228753Smm#endif 81228753Smmstatic ssize_t memory_read(struct archive *, void *, const void **buff); 82228753Smmstatic ssize_t memory_write(struct archive *, void *, const void *, size_t); 83228753Smm 84228753Smm 85228753Smmstatic ssize_t 86228753Smmmemory_write(struct archive *a, void *_private, const void *buff, size_t size) 87228753Smm{ 88228753Smm struct memdata *private = _private; 89228753Smm struct memblock *block; 90228753Smm 91228753Smm (void)a; 92228753Smm 93228753Smm /* 94228753Smm * Since libarchive tries to behave in a zero-copy manner, if 95228753Smm * you give a pointer to filedata to the library, a pointer 96228753Smm * into that data will (usually) pop out here. This way, we 97228753Smm * can tell the difference between filedata and library header 98228753Smm * and metadata. 99228753Smm */ 100228753Smm if ((const char *)filedata <= (const char *)buff 101228753Smm && (const char *)buff < (const char *)filedata + filedatasize) { 102228753Smm /* We don't need to store a block of file data. */ 103228753Smm private->last->filebytes += (int64_t)size; 104228753Smm } else { 105228753Smm /* Yes, we're assuming the very first write is metadata. */ 106228753Smm /* It's header or metadata, copy and save it. */ 107228753Smm block = (struct memblock *)malloc(sizeof(*block)); 108228753Smm memset(block, 0, sizeof(*block)); 109228753Smm block->size = size; 110228753Smm block->buff = malloc(size); 111228753Smm memcpy(block->buff, buff, size); 112228753Smm if (private->last == NULL) { 113228753Smm private->first = private->last = block; 114228753Smm } else { 115228753Smm private->last->next = block; 116228753Smm private->last = block; 117228753Smm } 118228753Smm block->next = NULL; 119228753Smm } 120228753Smm return ((long)size); 121228753Smm} 122228753Smm 123228753Smmstatic ssize_t 124228753Smmmemory_read(struct archive *a, void *_private, const void **buff) 125228753Smm{ 126228753Smm struct memdata *private = _private; 127228753Smm struct memblock *block; 128228753Smm ssize_t size; 129228753Smm 130228753Smm (void)a; 131228753Smm 132228753Smm free(private->buff); 133228753Smm private->buff = NULL; 134228753Smm if (private->first == NULL) { 135228753Smm private->last = NULL; 136228753Smm return (ARCHIVE_EOF); 137228753Smm } 138228753Smm if (private->filebytes > 0) { 139228753Smm /* 140228753Smm * We're returning file bytes, simulate it by 141228753Smm * passing blocks from the template data. 142228753Smm */ 143228753Smm if (private->filebytes > (int64_t)filedatasize) 144228753Smm size = (ssize_t)filedatasize; 145228753Smm else 146228753Smm size = (ssize_t)private->filebytes; 147228753Smm private->filebytes -= size; 148228753Smm *buff = filedata; 149228753Smm } else { 150228753Smm /* 151228753Smm * We need to get some real data to return. 152228753Smm */ 153228753Smm block = private->first; 154228753Smm private->first = block->next; 155228753Smm size = (ssize_t)block->size; 156228753Smm if (block->buff != NULL) { 157228753Smm private->buff = block->buff; 158228753Smm *buff = block->buff; 159228753Smm } else { 160228753Smm private->buff = NULL; 161228753Smm *buff = filedata; 162228753Smm } 163228753Smm private->filebytes = block->filebytes; 164228753Smm free(block); 165228753Smm } 166228753Smm return (size); 167228753Smm} 168228753Smm 169228753Smm 170228753Smm#if ARCHIVE_VERSION_NUMBER < 2000000 171228753Smmstatic ssize_t 172228753Smmmemory_read_skip(struct archive *a, void *private, size_t skip) 173228753Smm{ 174228753Smm (void)a; /* UNUSED */ 175228753Smm (void)private; /* UNUSED */ 176228753Smm (void)skip; /* UNUSED */ 177228753Smm return (0); 178228753Smm} 179228753Smm#else 180228753Smmstatic off_t 181228753Smmmemory_read_skip(struct archive *a, void *_private, off_t skip) 182228753Smm{ 183228753Smm struct memdata *private = _private; 184228753Smm 185228753Smm (void)a; 186228753Smm 187228753Smm if (private->first == NULL) { 188228753Smm private->last = NULL; 189228753Smm return (0); 190228753Smm } 191228753Smm if (private->filebytes > 0) { 192228753Smm if (private->filebytes < skip) 193228753Smm skip = (off_t)private->filebytes; 194228753Smm private->filebytes -= skip; 195228753Smm } else { 196228753Smm skip = 0; 197228753Smm } 198228753Smm return (skip); 199228753Smm} 200228753Smm#endif 201228753Smm 202228753SmmDEFINE_TEST(test_tar_large) 203228753Smm{ 204228753Smm /* The sizes of the entries we're going to generate. */ 205228753Smm static int64_t tests[] = { 206228753Smm /* Test for 32-bit signed overflow. */ 207228753Smm 2 * GB - 1, 2 * GB, 2 * GB + 1, 208228753Smm /* Test for 32-bit unsigned overflow. */ 209228753Smm 4 * GB - 1, 4 * GB, 4 * GB + 1, 210228753Smm /* 8GB is the "official" max for ustar. */ 211228753Smm 8 * GB - 1, 8 * GB, 8 * GB + 1, 212228753Smm /* Bend ustar a tad and you can get 64GB (12 octal digits). */ 213228753Smm 64 * GB - 1, 64 * GB, 214228753Smm /* And larger entries that require non-ustar extensions. */ 215228753Smm 256 * GB, 1 * TB, 0 }; 216228753Smm int i; 217228753Smm char namebuff[64]; 218228753Smm struct memdata memdata; 219228753Smm struct archive_entry *ae; 220228753Smm struct archive *a; 221228753Smm int64_t filesize; 222228753Smm size_t writesize; 223228753Smm 224228753Smm filedatasize = (size_t)(1 * MB); 225228753Smm filedata = malloc(filedatasize); 226228753Smm memset(filedata, 0xAA, filedatasize); 227228753Smm memset(&memdata, 0, sizeof(memdata)); 228228753Smm 229228753Smm /* 230228753Smm * Open an archive for writing. 231228753Smm */ 232228753Smm a = archive_write_new(); 233228753Smm archive_write_set_format_pax_restricted(a); 234228753Smm archive_write_set_bytes_per_block(a, 0); /* No buffering. */ 235228753Smm archive_write_open(a, &memdata, NULL, memory_write, NULL); 236228753Smm 237228753Smm /* 238228753Smm * Write a series of large files to it. 239228753Smm */ 240228753Smm for (i = 0; tests[i] != 0; i++) { 241228753Smm assert((ae = archive_entry_new()) != NULL); 242228753Smm sprintf(namebuff, "file_%d", i); 243228753Smm archive_entry_copy_pathname(ae, namebuff); 244228753Smm archive_entry_set_mode(ae, S_IFREG | 0755); 245228753Smm filesize = tests[i]; 246228753Smm 247228753Smm archive_entry_set_size(ae, filesize); 248228753Smm 249228753Smm assertA(0 == archive_write_header(a, ae)); 250228753Smm archive_entry_free(ae); 251228753Smm 252228753Smm /* 253228753Smm * Write the actual data to the archive. 254228753Smm */ 255228753Smm while (filesize > 0) { 256228753Smm writesize = filedatasize; 257228753Smm if ((int64_t)writesize > filesize) 258228753Smm writesize = (size_t)filesize; 259228753Smm assertA((int)writesize 260228753Smm == archive_write_data(a, filedata, writesize)); 261228753Smm filesize -= writesize; 262228753Smm } 263228753Smm } 264228753Smm 265228753Smm assert((ae = archive_entry_new()) != NULL); 266228753Smm archive_entry_copy_pathname(ae, "lastfile"); 267228753Smm archive_entry_set_mode(ae, S_IFREG | 0755); 268228753Smm assertA(0 == archive_write_header(a, ae)); 269228753Smm archive_entry_free(ae); 270228753Smm 271228753Smm 272228753Smm /* Close out the archive. */ 273228753Smm assertA(0 == archive_write_close(a)); 274228753Smm#if ARCHIVE_VERSION_NUMBER < 2000000 275228753Smm archive_write_finish(a); 276228753Smm#else 277228753Smm assertA(0 == archive_write_finish(a)); 278228753Smm#endif 279228753Smm 280228753Smm /* 281228753Smm * Open the same archive for reading. 282228753Smm */ 283228753Smm a = archive_read_new(); 284228753Smm archive_read_support_format_tar(a); 285228753Smm archive_read_open2(a, &memdata, NULL, 286228753Smm memory_read, memory_read_skip, NULL); 287228753Smm 288228753Smm /* 289228753Smm * Read entries back. 290228753Smm */ 291228753Smm for (i = 0; tests[i] > 0; i++) { 292228753Smm assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 293228753Smm sprintf(namebuff, "file_%d", i); 294228753Smm assertEqualString(namebuff, archive_entry_pathname(ae)); 295228753Smm assert(tests[i] == archive_entry_size(ae)); 296228753Smm } 297228753Smm assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 298228753Smm assertEqualString("lastfile", archive_entry_pathname(ae)); 299228753Smm 300228753Smm assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); 301228753Smm 302228753Smm /* Close out the archive. */ 303228753Smm assertA(0 == archive_read_close(a)); 304228753Smm#if ARCHIVE_VERSION_NUMBER < 2000000 305228753Smm archive_read_finish(a); 306228753Smm#else 307228753Smm assertA(0 == archive_read_finish(a)); 308228753Smm#endif 309228753Smm 310228753Smm free(memdata.buff); 311228753Smm free(filedata); 312228753Smm} 313