test_sparse_basic.c revision 231200 
1/*-
2 * Copyright (c) 2010 Michihiro NAKAJIMA
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25#include "test.h"
26__FBSDID("$FreeBSD$");
27
28#ifdef HAVE_SYS_IOCTL_H
29#include <sys/ioctl.h>
30#endif
31#ifdef HAVE_SYS_PARAM_H
32#include <sys/param.h>
33#endif
34#ifdef HAVE_SYS_UTSNAME_H
35#include <sys/utsname.h>
36#endif
37#ifdef HAVE_FCNTL_H
38#include <fcntl.h>
39#endif
40#ifdef HAVE_LIMITS_H
41#include <limits.h>
42#endif
43#ifdef HAVE_UNISTD_H
44#include <unistd.h>
45#endif
46#ifdef HAVE_LINUX_FIEMAP_H
47#include <linux/fiemap.h>
48#endif
49#ifdef HAVE_LINUX_FS_H
50#include <linux/fs.h>
51#endif
52
53/*
54 * NOTE: On FreeBSD and Solaris, this test needs ZFS.
55 * You may should perfom this test as
56 * 'TMPDIR=<a directory on the ZFS> libarchive_test'.
57 */
58
59struct sparse {
60	enum { DATA, HOLE, END } type;
61	size_t	size;
62};
63
64static void create_sparse_file(const char *, const struct sparse *);
65
66#if defined(_WIN32) && !defined(__CYGWIN__)
67#include <winioctl.h>
68/*
69 * Create a sparse file on Windows.
70 */
71
72#if !defined(PATH_MAX)
73#define	PATH_MAX	MAX_PATH
74#endif
75#if !defined(__BORLANDC__)
76#define getcwd _getcwd
77#endif
78
79static int
80is_sparse_supported(const char *path)
81{
82	char root[MAX_PATH+1];
83	char vol[MAX_PATH+1];
84	char sys[MAX_PATH+1];
85	DWORD flags;
86	BOOL r;
87
88	strncpy(root, path, sizeof(root)-1);
89	if (((root[0] >= 'c' && root[0] <= 'z') ||
90	    (root[0] >= 'C' && root[0] <= 'Z')) &&
91		root[1] == ':' &&
92	    (root[2] == '\\' || root[2] == '/'))
93		root[3] = '\0';
94	else
95		return (0);
96	assertEqualInt((r = GetVolumeInformation(root, vol,
97	    sizeof(vol), NULL, NULL, &flags, sys, sizeof(sys))), 1);
98	return (r != 0 && (flags & FILE_SUPPORTS_SPARSE_FILES) != 0);
99}
100
101static void
102create_sparse_file(const char *path, const struct sparse *s)
103{
104	char buff[1024];
105	HANDLE handle;
106	DWORD dmy;
107
108	memset(buff, ' ', sizeof(buff));
109
110	handle = CreateFileA(path, GENERIC_WRITE, 0,
111	    NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL,
112	    NULL);
113	assert(handle != INVALID_HANDLE_VALUE);
114	assert(DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0,
115	    NULL, 0, &dmy, NULL) != 0);
116	while (s->type != END) {
117		if (s->type == HOLE) {
118			LARGE_INTEGER distance;
119
120			distance.QuadPart = s->size;
121			assert(SetFilePointerEx(handle, distance,
122			    NULL, FILE_CURRENT) != 0);
123		} else {
124			DWORD w, wr, size;
125
126			size = s->size;
127			while (size) {
128				if (size > sizeof(buff))
129					w = sizeof(buff);
130				else
131					w = size;
132				assert(WriteFile(handle, buff, w, &wr, NULL) != 0);
133				size -= wr;
134			}
135		}
136		s++;
137	}
138	assertEqualInt(CloseHandle(handle), 1);
139}
140
141#else
142
143#if defined(_PC_MIN_HOLE_SIZE)
144
145/*
146 * FreeBSD and Solaris can detect 'hole' of a sparse file
147 * through lseek(HOLE) on ZFS. (UFS does not support yet)
148 */
149
150static int
151is_sparse_supported(const char *path)
152{
153	return (pathconf(path, _PC_MIN_HOLE_SIZE) > 0);
154}
155
156#elif defined(__linux__)&& defined(HAVE_LINUX_FIEMAP_H)
157
158/*
159 * FIEMAP, which can detect 'hole' of a sparse file, has
160 * been supported from 2.6.28
161 */
162
163static int
164is_sparse_supported(const char *path)
165{
166	const struct sparse sparse_file[] = {
167 		/* This hole size is too small to create a sparse
168		 * files for almost filesystem. */
169		{ HOLE,	 1024 }, { DATA, 10240 },
170		{ END,	0 }
171	};
172	struct utsname ut;
173	char *p, *e;
174	long d;
175	int fd, r;
176	struct fiemap *fm;
177	char buff[1024];
178	const char *testfile = "can_sparse";
179
180	memset(&ut, 0, sizeof(ut));
181	assertEqualInt(uname(&ut), 0);
182	p = ut.release;
183	d = strtol(p, &e, 10);
184	if (d < 2 || *e != '.')
185		return (0);
186	if (d == 2) {
187		p = e + 1;
188		d = strtol(p, &e, 10);
189		if (d < 6 || *e != '.')
190			return (0);
191		p = e + 1;
192		d = strtol(p, NULL, 10);
193		if (d < 28)
194			return (0);
195	}
196	create_sparse_file(testfile, sparse_file);
197	fd = open(testfile,  O_RDWR);
198	if (fd < 0)
199		return (0);
200	fm = (struct fiemap *)buff;
201	fm->fm_start = 0;
202	fm->fm_length = ~0ULL;;
203	fm->fm_flags = FIEMAP_FLAG_SYNC;
204	fm->fm_extent_count = (sizeof(buff) - sizeof(*fm))/
205		sizeof(struct fiemap_extent);
206	r = ioctl(fd, FS_IOC_FIEMAP, fm);
207	if (r < 0 && (errno == ENOTTY || errno == EOPNOTSUPP))
208		return (0);/* Not supported. */
209	close(fd);
210	unlink(testfile);
211	return (1);
212}
213
214#else
215
216/*
217 * Other system may do not have the API such as lseek(HOLE),
218 * which detect 'hole' of a sparse file.
219 */
220
221static int
222is_sparse_supported(const char *path)
223{
224	return (0);
225}
226
227#endif
228
229/*
230 * Create a sparse file on POSIX like system.
231 */
232
233static void
234create_sparse_file(const char *path, const struct sparse *s)
235{
236	char buff[1024];
237	int fd;
238
239	memset(buff, ' ', sizeof(buff));
240	assert((fd = open(path, O_CREAT | O_WRONLY, 0600)) != -1);
241	while (s->type != END) {
242		if (s->type == HOLE) {
243			assert(lseek(fd, s->size, SEEK_CUR) != (off_t)-1);
244		} else {
245			size_t w, size;
246
247			size = s->size;
248			while (size) {
249				if (size > sizeof(buff))
250					w = sizeof(buff);
251				else
252					w = size;
253				assert(write(fd, buff, w) != (ssize_t)-1);
254				size -= w;
255			}
256		}
257		s++;
258	}
259	close(fd);
260}
261
262#endif
263
264/*
265 * Sparse test with directory traversals.
266 */
267static void
268verify_sparse_file(struct archive *a, const char *path,
269    const struct sparse *sparse, int blocks)
270{
271	struct archive_entry *ae;
272	const void *buff;
273	size_t bytes_read;
274	int64_t offset;
275	int64_t total;
276	int data_blocks, hole;
277
278	create_sparse_file(path, sparse);
279	assert((ae = archive_entry_new()) != NULL);
280	assertEqualIntA(a, ARCHIVE_OK, archive_read_disk_open(a, path));
281	assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header2(a, ae));
282	/* Verify the number of holes only, not its offset nor its
283	 * length because those alignments are deeply dependence on
284	 * its filesystem. */
285	assertEqualInt(blocks, archive_entry_sparse_count(ae));
286	total = 0;
287	data_blocks = 0;
288	hole = 0;
289	while (ARCHIVE_OK == archive_read_data_block(a, &buff, &bytes_read,
290	    &offset)) {
291		if (offset > total || offset == 0) {
292			if (offset > total)
293				hole = 1;
294			data_blocks++;
295		}
296		total = offset + bytes_read;
297	}
298	if (!hole && data_blocks == 1)
299		data_blocks = 0;/* There are no holes */
300	assertEqualInt(blocks, data_blocks);
301
302	assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a));
303	archive_entry_free(ae);
304}
305
306#if defined(_WIN32) && !defined(__CYGWIN__)
307#define	close		_close
308#define	open		_open
309#endif
310
311/*
312 * Sparse test without directory traversals.
313 */
314static void
315verify_sparse_file2(struct archive *a, const char *path,
316    const struct sparse *sparse, int blocks, int preopen)
317{
318	struct archive_entry *ae;
319	int fd;
320
321	(void)sparse; /* UNUSED */
322	assert((ae = archive_entry_new()) != NULL);
323	archive_entry_set_pathname(ae, path);
324	if (preopen)
325		fd = open(path, O_RDONLY | O_BINARY);
326	else
327		fd = -1;
328	assertEqualIntA(a, ARCHIVE_OK,
329	    archive_read_disk_entry_from_file(a, ae, fd, NULL));
330	if (fd >= 0)
331		close(fd);
332	/* Verify the number of holes only, not its offset nor its
333	 * length because those alignments are deeply dependence on
334	 * its filesystem. */
335	assertEqualInt(blocks, archive_entry_sparse_count(ae));
336	archive_entry_free(ae);
337}
338
339static void
340test_sparse_whole_file_data()
341{
342	struct archive_entry *ae;
343	int64_t offset;
344	int i;
345
346	assert((ae = archive_entry_new()) != NULL);
347	archive_entry_set_size(ae, 1024*10);
348
349	/*
350	 * Add sparse block data up to the file size.
351	 */
352	offset = 0;
353	for (i = 0; i < 10; i++) {
354		archive_entry_sparse_add_entry(ae, offset, 1024);
355		offset += 1024;
356	}
357
358	failure("There should be no sparse");
359	assertEqualInt(0, archive_entry_sparse_count(ae));
360	archive_entry_free(ae);
361}
362
363DEFINE_TEST(test_sparse_basic)
364{
365	char *cwd;
366	struct archive *a;
367	/*
368	 * The alignment of the hole of sparse files deeply depends
369	 * on filesystem. In my experience, sparse_file2 test with
370	 * 204800 bytes hole size did not pass on ZFS and the result
371	 * of that test seemed the size was too small, thus you should
372	 * keep a hole size more than 409600 bytes to pass this test
373	 * on all platform.
374	 */
375	const struct sparse sparse_file0[] = {
376		{ DATA,	 1024 }, { HOLE,   2048000 },
377		{ DATA,	 2048 }, { HOLE,   2048000 },
378		{ DATA,	 4096 }, { HOLE,  20480000 },
379		{ DATA,	 8192 }, { HOLE, 204800000 },
380		{ DATA,     1 }, { END,	0 }
381	};
382	const struct sparse sparse_file1[] = {
383		{ HOLE,	409600 }, { DATA, 1 },
384		{ HOLE,	409600 }, { DATA, 1 },
385		{ HOLE,	409600 }, { END,  0 }
386	};
387	const struct sparse sparse_file2[] = {
388		{ HOLE,	409600 * 1 }, { DATA, 1024 },
389		{ HOLE,	409600 * 2 }, { DATA, 1024 },
390		{ HOLE,	409600 * 3 }, { DATA, 1024 },
391		{ HOLE,	409600 * 4 }, { DATA, 1024 },
392		{ HOLE,	409600 * 5 }, { DATA, 1024 },
393		{ HOLE,	409600 * 6 }, { DATA, 1024 },
394		{ HOLE,	409600 * 7 }, { DATA, 1024 },
395		{ HOLE,	409600 * 8 }, { DATA, 1024 },
396		{ HOLE,	409600 * 9 }, { DATA, 1024 },
397		{ HOLE,	409600 * 10}, { DATA, 1024 },/* 10 */
398		{ HOLE,	409600 * 1 }, { DATA, 1024 * 1 },
399		{ HOLE,	409600 * 2 }, { DATA, 1024 * 2 },
400		{ HOLE,	409600 * 3 }, { DATA, 1024 * 3 },
401		{ HOLE,	409600 * 4 }, { DATA, 1024 * 4 },
402		{ HOLE,	409600 * 5 }, { DATA, 1024 * 5 },
403		{ HOLE,	409600 * 6 }, { DATA, 1024 * 6 },
404		{ HOLE,	409600 * 7 }, { DATA, 1024 * 7 },
405		{ HOLE,	409600 * 8 }, { DATA, 1024 * 8 },
406		{ HOLE,	409600 * 9 }, { DATA, 1024 * 9 },
407		{ HOLE,	409600 * 10}, { DATA, 1024 * 10},/* 20 */
408		{ END,	0 }
409	};
410	const struct sparse sparse_file3[] = {
411 		/* This hole size is too small to create a sparse
412		 * files for almost filesystem. */
413		{ HOLE,	 1024 }, { DATA, 10240 },
414		{ END,	0 }
415	};
416
417	/*
418	 * Test for the case that sparse data indicates just the whole file
419	 * data.
420	 */
421	test_sparse_whole_file_data();
422
423	/* Check if the filesystem where CWD on can
424	 * report the number of the holes of a sparse file. */
425#ifdef PATH_MAX
426	cwd = getcwd(NULL, PATH_MAX);/* Solaris getcwd needs the size. */
427#else
428	cwd = getcwd(NULL, 0);
429#endif
430	if (!assert(cwd != NULL))
431		return;
432	if (!is_sparse_supported(cwd)) {
433		free(cwd);
434		skipping("This filesystem or platform do not support "
435		    "the reporting of the holes of a sparse file through "
436		    "API such as lseek(HOLE)");
437		return;
438	}
439
440	/*
441	 * Get sparse data through directory traversals.
442	 */
443	assert((a = archive_read_disk_new()) != NULL);
444
445	verify_sparse_file(a, "file0", sparse_file0, 5);
446	verify_sparse_file(a, "file1", sparse_file1, 2);
447	verify_sparse_file(a, "file2", sparse_file2, 20);
448	verify_sparse_file(a, "file3", sparse_file3, 0);
449
450	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
451
452	/*
453	 * Get sparse data through archive_read_disk_entry_from_file().
454	 */
455	assert((a = archive_read_disk_new()) != NULL);
456
457	verify_sparse_file2(a, "file0", sparse_file0, 5, 0);
458	verify_sparse_file2(a, "file0", sparse_file0, 5, 1);
459
460	assertEqualInt(ARCHIVE_OK, archive_read_free(a));
461	free(cwd);
462}
463