1///////////////////////////////////////////////////////////////////////////////
2//
3/// \file       04_compress_easy_mt.c
4/// \brief      Compress in multi-call mode using LZMA2 in multi-threaded mode
5///
6/// Usage:      ./04_compress_easy_mt < INFILE > OUTFILE
7///
8/// Example:    ./04_compress_easy_mt < foo > foo.xz
9//
10//  Author:     Lasse Collin
11//
12//  This file has been put into the public domain.
13//  You can do whatever you want with this file.
14//
15///////////////////////////////////////////////////////////////////////////////
16
17#include <stdbool.h>
18#include <stdlib.h>
19#include <stdio.h>
20#include <string.h>
21#include <errno.h>
22#include <lzma.h>
23
24
25static bool
26init_encoder(lzma_stream *strm)
27{
28	// The threaded encoder takes the options as pointer to
29	// a lzma_mt structure.
30	lzma_mt mt = {
31		// No flags are needed.
32		.flags = 0,
33
34		// Let liblzma determine a sane block size.
35		.block_size = 0,
36
37		// Use no timeout for lzma_code() calls by setting timeout
38		// to zero. That is, sometimes lzma_code() might block for
39		// a long time (from several seconds to even minutes).
40		// If this is not OK, for example due to progress indicator
41		// needing updates, specify a timeout in milliseconds here.
42		// See the documentation of lzma_mt in lzma/container.h for
43		// information how to choose a reasonable timeout.
44		.timeout = 0,
45
46		// Use the default preset (6) for LZMA2.
47		// To use a preset, filters must be set to NULL.
48		.preset = LZMA_PRESET_DEFAULT,
49		.filters = NULL,
50
51		// Use CRC64 for integrity checking. See also
52		// 01_compress_easy.c about choosing the integrity check.
53		.check = LZMA_CHECK_CRC64,
54	};
55
56	// Detect how many threads the CPU supports.
57	mt.threads = lzma_cputhreads();
58
59	// If the number of CPU cores/threads cannot be detected,
60	// use one thread. Note that this isn't the same as the normal
61	// single-threaded mode as this will still split the data into
62	// blocks and use more RAM than the normal single-threaded mode.
63	// You may want to consider using lzma_easy_encoder() or
64	// lzma_stream_encoder() instead of lzma_stream_encoder_mt() if
65	// lzma_cputhreads() returns 0 or 1.
66	if (mt.threads == 0)
67		mt.threads = 1;
68
69	// If the number of CPU cores/threads exceeds threads_max,
70	// limit the number of threads to keep memory usage lower.
71	// The number 8 is arbitrarily chosen and may be too low or
72	// high depending on the compression preset and the computer
73	// being used.
74	//
75	// FIXME: A better way could be to check the amount of RAM
76	// (or available RAM) and use lzma_stream_encoder_mt_memusage()
77	// to determine if the number of threads should be reduced.
78	const uint32_t threads_max = 8;
79	if (mt.threads > threads_max)
80		mt.threads = threads_max;
81
82	// Initialize the threaded encoder.
83	lzma_ret ret = lzma_stream_encoder_mt(strm, &mt);
84
85	if (ret == LZMA_OK)
86		return true;
87
88	const char *msg;
89	switch (ret) {
90	case LZMA_MEM_ERROR:
91		msg = "Memory allocation failed";
92		break;
93
94	case LZMA_OPTIONS_ERROR:
95		// We are no longer using a plain preset so this error
96		// message has been edited accordingly compared to
97		// 01_compress_easy.c.
98		msg = "Specified filter chain is not supported";
99		break;
100
101	case LZMA_UNSUPPORTED_CHECK:
102		msg = "Specified integrity check is not supported";
103		break;
104
105	default:
106		msg = "Unknown error, possibly a bug";
107		break;
108	}
109
110	fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n",
111			msg, ret);
112	return false;
113}
114
115
116// This function is identical to the one in 01_compress_easy.c.
117static bool
118compress(lzma_stream *strm, FILE *infile, FILE *outfile)
119{
120	lzma_action action = LZMA_RUN;
121
122	uint8_t inbuf[BUFSIZ];
123	uint8_t outbuf[BUFSIZ];
124
125	strm->next_in = NULL;
126	strm->avail_in = 0;
127	strm->next_out = outbuf;
128	strm->avail_out = sizeof(outbuf);
129
130	while (true) {
131		if (strm->avail_in == 0 && !feof(infile)) {
132			strm->next_in = inbuf;
133			strm->avail_in = fread(inbuf, 1, sizeof(inbuf),
134					infile);
135
136			if (ferror(infile)) {
137				fprintf(stderr, "Read error: %s\n",
138						strerror(errno));
139				return false;
140			}
141
142			if (feof(infile))
143				action = LZMA_FINISH;
144		}
145
146		lzma_ret ret = lzma_code(strm, action);
147
148		if (strm->avail_out == 0 || ret == LZMA_STREAM_END) {
149			size_t write_size = sizeof(outbuf) - strm->avail_out;
150
151			if (fwrite(outbuf, 1, write_size, outfile)
152					!= write_size) {
153				fprintf(stderr, "Write error: %s\n",
154						strerror(errno));
155				return false;
156			}
157
158			strm->next_out = outbuf;
159			strm->avail_out = sizeof(outbuf);
160		}
161
162		if (ret != LZMA_OK) {
163			if (ret == LZMA_STREAM_END)
164				return true;
165
166			const char *msg;
167			switch (ret) {
168			case LZMA_MEM_ERROR:
169				msg = "Memory allocation failed";
170				break;
171
172			case LZMA_DATA_ERROR:
173				msg = "File size limits exceeded";
174				break;
175
176			default:
177				msg = "Unknown error, possibly a bug";
178				break;
179			}
180
181			fprintf(stderr, "Encoder error: %s (error code %u)\n",
182					msg, ret);
183			return false;
184		}
185	}
186}
187
188
189extern int
190main(void)
191{
192	lzma_stream strm = LZMA_STREAM_INIT;
193
194	bool success = init_encoder(&strm);
195	if (success)
196		success = compress(&strm, stdin, stdout);
197
198	lzma_end(&strm);
199
200	if (fclose(stdout)) {
201		fprintf(stderr, "Write error: %s\n", strerror(errno));
202		success = false;
203	}
204
205	return success ? EXIT_SUCCESS : EXIT_FAILURE;
206}
207