1228753Smm/*- 2228753Smm * Copyright (c) 2003-2007 Tim Kientzle 3228753Smm * All rights reserved. 4228753Smm * 5228753Smm * Redistribution and use in source and binary forms, with or without 6228753Smm * modification, are permitted provided that the following conditions 7228753Smm * are met: 8228753Smm * 1. Redistributions of source code must retain the above copyright 9228753Smm * notice, this list of conditions and the following disclaimer. 10228753Smm * 2. Redistributions in binary form must reproduce the above copyright 11228753Smm * notice, this list of conditions and the following disclaimer in the 12228753Smm * documentation and/or other materials provided with the distribution. 13228753Smm * 14228753Smm * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 15228753Smm * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16228753Smm * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17228753Smm * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 18228753Smm * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19228753Smm * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20228753Smm * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21228753Smm * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22228753Smm * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23228753Smm * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24228753Smm */ 25228753Smm#include "test.h" 26228763Smm__FBSDID("$FreeBSD$"); 27228753Smm 28228753Smm#include <errno.h> 29228753Smm#include <stdlib.h> 30228753Smm#include <string.h> 31228753Smm 32228753Smm/* 33228753Smm * This is a somewhat tricky test that verifies the ability to 34228753Smm * write and read very large entries to tar archives. It 35228753Smm * writes entries from 2GB up to 1TB to an archive in memory. 36228753Smm * The memory storage here carefully avoids actually storing 37228753Smm * any part of the file bodies, so it runs very quickly and requires 38228753Smm * very little memory. If you're willing to wait a few minutes, 39228753Smm * you should be able to exercise petabyte entries with this code. 40228753Smm */ 41228753Smm 42228753Smm/* 43228753Smm * Each file is built up by duplicating the following block. 44228753Smm */ 45228753Smmstatic size_t filedatasize; 46228753Smmstatic void *filedata; 47228753Smm 48228753Smm/* 49228753Smm * We store the archive as blocks of data generated by libarchive, 50228753Smm * each possibly followed by bytes of file data. 51228753Smm */ 52228753Smmstruct memblock { 53228753Smm struct memblock *next; 54228753Smm size_t size; 55228753Smm void *buff; 56228753Smm int64_t filebytes; 57228753Smm}; 58228753Smm 59228753Smm/* 60228753Smm * The total memory store is just a list of memblocks plus 61228753Smm * some accounting overhead. 62228753Smm */ 63228753Smmstruct memdata { 64228753Smm int64_t filebytes; 65228753Smm void *buff; 66228753Smm struct memblock *first; 67228753Smm struct memblock *last; 68228753Smm}; 69228753Smm 70228753Smm/* The following size definitions simplify things below. */ 71228753Smm#define KB ((int64_t)1024) 72228753Smm#define MB ((int64_t)1024 * KB) 73228753Smm#define GB ((int64_t)1024 * MB) 74228753Smm#define TB ((int64_t)1024 * GB) 75228753Smm 76232153Smmstatic int64_t memory_read_skip(struct archive *, void *, int64_t request); 77228753Smmstatic ssize_t memory_read(struct archive *, void *, const void **buff); 78228753Smmstatic ssize_t memory_write(struct archive *, void *, const void *, size_t); 79228753Smm 80228753Smm 81228753Smmstatic ssize_t 82228753Smmmemory_write(struct archive *a, void *_private, const void *buff, size_t size) 83228753Smm{ 84228753Smm struct memdata *private = _private; 85228753Smm struct memblock *block; 86228753Smm 87228753Smm (void)a; 88228753Smm 89228753Smm /* 90228753Smm * Since libarchive tries to behave in a zero-copy manner, if 91228753Smm * you give a pointer to filedata to the library, a pointer 92228753Smm * into that data will (usually) pop out here. This way, we 93228753Smm * can tell the difference between filedata and library header 94228753Smm * and metadata. 95228753Smm */ 96228753Smm if ((const char *)filedata <= (const char *)buff 97228753Smm && (const char *)buff < (const char *)filedata + filedatasize) { 98228753Smm /* We don't need to store a block of file data. */ 99228753Smm private->last->filebytes += (int64_t)size; 100228753Smm } else { 101228753Smm /* Yes, we're assuming the very first write is metadata. */ 102228753Smm /* It's header or metadata, copy and save it. */ 103228753Smm block = (struct memblock *)malloc(sizeof(*block)); 104228753Smm memset(block, 0, sizeof(*block)); 105228753Smm block->size = size; 106228753Smm block->buff = malloc(size); 107228753Smm memcpy(block->buff, buff, size); 108228753Smm if (private->last == NULL) { 109228753Smm private->first = private->last = block; 110228753Smm } else { 111228753Smm private->last->next = block; 112228753Smm private->last = block; 113228753Smm } 114228753Smm block->next = NULL; 115228753Smm } 116228753Smm return ((long)size); 117228753Smm} 118228753Smm 119228753Smmstatic ssize_t 120228753Smmmemory_read(struct archive *a, void *_private, const void **buff) 121228753Smm{ 122228753Smm struct memdata *private = _private; 123228753Smm struct memblock *block; 124228753Smm ssize_t size; 125228753Smm 126228753Smm (void)a; 127228753Smm 128228753Smm free(private->buff); 129228753Smm private->buff = NULL; 130228753Smm if (private->first == NULL) { 131228753Smm private->last = NULL; 132228753Smm return (ARCHIVE_EOF); 133228753Smm } 134228753Smm if (private->filebytes > 0) { 135228753Smm /* 136228753Smm * We're returning file bytes, simulate it by 137228753Smm * passing blocks from the template data. 138228753Smm */ 139228753Smm if (private->filebytes > (int64_t)filedatasize) 140228753Smm size = (ssize_t)filedatasize; 141228753Smm else 142228753Smm size = (ssize_t)private->filebytes; 143228753Smm private->filebytes -= size; 144228753Smm *buff = filedata; 145228753Smm } else { 146228753Smm /* 147228753Smm * We need to get some real data to return. 148228753Smm */ 149228753Smm block = private->first; 150228753Smm private->first = block->next; 151228753Smm size = (ssize_t)block->size; 152228753Smm if (block->buff != NULL) { 153228753Smm private->buff = block->buff; 154228753Smm *buff = block->buff; 155228753Smm } else { 156228753Smm private->buff = NULL; 157228753Smm *buff = filedata; 158228753Smm } 159228753Smm private->filebytes = block->filebytes; 160228753Smm free(block); 161228753Smm } 162228753Smm return (size); 163228753Smm} 164228753Smm 165228753Smm 166232153Smmstatic int64_t 167232153Smmmemory_read_skip(struct archive *a, void *_private, int64_t skip) 168228753Smm{ 169228753Smm struct memdata *private = _private; 170228753Smm 171228753Smm (void)a; 172228753Smm 173228753Smm if (private->first == NULL) { 174228753Smm private->last = NULL; 175228753Smm return (0); 176228753Smm } 177228753Smm if (private->filebytes > 0) { 178228753Smm if (private->filebytes < skip) 179228753Smm skip = (off_t)private->filebytes; 180228753Smm private->filebytes -= skip; 181228753Smm } else { 182228753Smm skip = 0; 183228753Smm } 184228753Smm return (skip); 185228753Smm} 186228753Smm 187228753SmmDEFINE_TEST(test_tar_large) 188228753Smm{ 189228753Smm /* The sizes of the entries we're going to generate. */ 190228753Smm static int64_t tests[] = { 191228753Smm /* Test for 32-bit signed overflow. */ 192228753Smm 2 * GB - 1, 2 * GB, 2 * GB + 1, 193228753Smm /* Test for 32-bit unsigned overflow. */ 194228753Smm 4 * GB - 1, 4 * GB, 4 * GB + 1, 195228753Smm /* 8GB is the "official" max for ustar. */ 196228753Smm 8 * GB - 1, 8 * GB, 8 * GB + 1, 197228753Smm /* Bend ustar a tad and you can get 64GB (12 octal digits). */ 198228753Smm 64 * GB - 1, 64 * GB, 199228753Smm /* And larger entries that require non-ustar extensions. */ 200228753Smm 256 * GB, 1 * TB, 0 }; 201228753Smm int i; 202228753Smm char namebuff[64]; 203228753Smm struct memdata memdata; 204228753Smm struct archive_entry *ae; 205228753Smm struct archive *a; 206228753Smm int64_t filesize; 207228753Smm size_t writesize; 208228753Smm 209228753Smm filedatasize = (size_t)(1 * MB); 210228753Smm filedata = malloc(filedatasize); 211228753Smm memset(filedata, 0xAA, filedatasize); 212228753Smm memset(&memdata, 0, sizeof(memdata)); 213228753Smm 214228753Smm /* 215228753Smm * Open an archive for writing. 216228753Smm */ 217228753Smm a = archive_write_new(); 218228753Smm archive_write_set_format_pax_restricted(a); 219228753Smm archive_write_set_bytes_per_block(a, 0); /* No buffering. */ 220228753Smm archive_write_open(a, &memdata, NULL, memory_write, NULL); 221228753Smm 222228753Smm /* 223228753Smm * Write a series of large files to it. 224228753Smm */ 225228753Smm for (i = 0; tests[i] != 0; i++) { 226228753Smm assert((ae = archive_entry_new()) != NULL); 227228753Smm sprintf(namebuff, "file_%d", i); 228228753Smm archive_entry_copy_pathname(ae, namebuff); 229228753Smm archive_entry_set_mode(ae, S_IFREG | 0755); 230228753Smm filesize = tests[i]; 231228753Smm 232228753Smm archive_entry_set_size(ae, filesize); 233228753Smm 234228753Smm assertA(0 == archive_write_header(a, ae)); 235228753Smm archive_entry_free(ae); 236228753Smm 237228753Smm /* 238228753Smm * Write the actual data to the archive. 239228753Smm */ 240228753Smm while (filesize > 0) { 241228753Smm writesize = filedatasize; 242228753Smm if ((int64_t)writesize > filesize) 243228753Smm writesize = (size_t)filesize; 244228753Smm assertA((int)writesize 245228753Smm == archive_write_data(a, filedata, writesize)); 246228753Smm filesize -= writesize; 247228753Smm } 248228753Smm } 249228753Smm 250228753Smm assert((ae = archive_entry_new()) != NULL); 251228753Smm archive_entry_copy_pathname(ae, "lastfile"); 252228753Smm archive_entry_set_mode(ae, S_IFREG | 0755); 253228753Smm assertA(0 == archive_write_header(a, ae)); 254228753Smm archive_entry_free(ae); 255228753Smm 256228753Smm 257228753Smm /* Close out the archive. */ 258232153Smm assertEqualIntA(a, ARCHIVE_OK, archive_write_close(a)); 259232153Smm assertEqualInt(ARCHIVE_OK, archive_write_free(a)); 260228753Smm 261228753Smm /* 262228753Smm * Open the same archive for reading. 263228753Smm */ 264228753Smm a = archive_read_new(); 265228753Smm archive_read_support_format_tar(a); 266228753Smm archive_read_open2(a, &memdata, NULL, 267228753Smm memory_read, memory_read_skip, NULL); 268228753Smm 269228753Smm /* 270228753Smm * Read entries back. 271228753Smm */ 272228753Smm for (i = 0; tests[i] > 0; i++) { 273228753Smm assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 274228753Smm sprintf(namebuff, "file_%d", i); 275228753Smm assertEqualString(namebuff, archive_entry_pathname(ae)); 276228753Smm assert(tests[i] == archive_entry_size(ae)); 277228753Smm } 278228753Smm assertEqualIntA(a, 0, archive_read_next_header(a, &ae)); 279228753Smm assertEqualString("lastfile", archive_entry_pathname(ae)); 280228753Smm 281228753Smm assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); 282228753Smm 283228753Smm /* Close out the archive. */ 284232153Smm assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); 285232153Smm assertEqualInt(ARCHIVE_OK, archive_read_free(a)); 286228753Smm 287228753Smm free(memdata.buff); 288228753Smm free(filedata); 289228753Smm} 290