Deleted Added
sdiff udiff text old ( 207842 ) new ( 213700 )
full compact
coder.c (207842) coder.c (213700)
1///////////////////////////////////////////////////////////////////////////////
2//
3/// \file coder.c
4/// \brief Compresses or uncompresses a file
5//
6// Author: Lasse Collin
7//
8// This file has been put into the public domain.

--- 8 unchanged lines hidden (view full) ---

17enum coder_init_ret {
18 CODER_INIT_NORMAL,
19 CODER_INIT_PASSTHRU,
20 CODER_INIT_ERROR,
21};
22
23
24enum operation_mode opt_mode = MODE_COMPRESS;
1///////////////////////////////////////////////////////////////////////////////
2//
3/// \file coder.c
4/// \brief Compresses or uncompresses a file
5//
6// Author: Lasse Collin
7//
8// This file has been put into the public domain.

--- 8 unchanged lines hidden (view full) ---

17enum coder_init_ret {
18 CODER_INIT_NORMAL,
19 CODER_INIT_PASSTHRU,
20 CODER_INIT_ERROR,
21};
22
23
24enum operation_mode opt_mode = MODE_COMPRESS;
25
26enum format_type opt_format = FORMAT_AUTO;
25enum format_type opt_format = FORMAT_AUTO;
26bool opt_auto_adjust = true;
27
28
29/// Stream used to communicate with liblzma
30static lzma_stream strm = LZMA_STREAM_INIT;
31
32/// Filters needed for all encoding all formats, and also decoding in raw data
33static lzma_filter filters[LZMA_FILTERS_MAX + 1];
34
35/// Input and output buffers
36static io_buf in_buf;
37static io_buf out_buf;
38
39/// Number of filters. Zero indicates that we are using a preset.
40static size_t filters_count = 0;
41
42/// Number of the preset (0-9)
43static size_t preset_number = 6;
44
27
28
29/// Stream used to communicate with liblzma
30static lzma_stream strm = LZMA_STREAM_INIT;
31
32/// Filters needed for all encoding all formats, and also decoding in raw data
33static lzma_filter filters[LZMA_FILTERS_MAX + 1];
34
35/// Input and output buffers
36static io_buf in_buf;
37static io_buf out_buf;
38
39/// Number of filters. Zero indicates that we are using a preset.
40static size_t filters_count = 0;
41
42/// Number of the preset (0-9)
43static size_t preset_number = 6;
44
45/// True if we should auto-adjust the compression settings to use less memory
46/// if memory usage limit is too low for the original settings.
47static bool auto_adjust = true;
48
49/// Indicate if no preset has been explicitly given. In that case, if we need
50/// to auto-adjust for lower memory usage, we won't print a warning.
51static bool preset_default = true;
52
53/// If a preset is used (no custom filter chain) and preset_extreme is true,
54/// a significantly slower compression is used to achieve slightly better
55/// compression ratio.
56static bool preset_extreme = false;
57
58/// Integrity check type
59static lzma_check check;
60

--- 9 unchanged lines hidden (view full) ---

70 return;
71}
72
73
74extern void
75coder_set_preset(size_t new_preset)
76{
77 preset_number = new_preset;
45/// If a preset is used (no custom filter chain) and preset_extreme is true,
46/// a significantly slower compression is used to achieve slightly better
47/// compression ratio.
48static bool preset_extreme = false;
49
50/// Integrity check type
51static lzma_check check;
52

--- 9 unchanged lines hidden (view full) ---

62 return;
63}
64
65
66extern void
67coder_set_preset(size_t new_preset)
68{
69 preset_number = new_preset;
78 preset_default = false;
70
71 // Setting a preset makes us forget a possibly defined custom
72 // filter chain.
73 while (filters_count > 0) {
74 --filters_count;
75 free(filters[filters_count].options);
76 filters[filters_count].options = NULL;
77 }
78
79 return;
80}
81
82
83extern void
84coder_set_extreme(void)
85{
86 preset_extreme = true;

--- 53 unchanged lines hidden (view full) ---

140 if (lzma_lzma_preset(&opt_lzma, preset_number))
141 message_bug();
142
143 // Use LZMA2 except with --format=lzma we use LZMA1.
144 filters[0].id = opt_format == FORMAT_LZMA
145 ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
146 filters[0].options = &opt_lzma;
147 filters_count = 1;
79 return;
80}
81
82
83extern void
84coder_set_extreme(void)
85{
86 preset_extreme = true;

--- 53 unchanged lines hidden (view full) ---

140 if (lzma_lzma_preset(&opt_lzma, preset_number))
141 message_bug();
142
143 // Use LZMA2 except with --format=lzma we use LZMA1.
144 filters[0].id = opt_format == FORMAT_LZMA
145 ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
146 filters[0].options = &opt_lzma;
147 filters_count = 1;
148 } else {
149 preset_default = false;
150 }
151
152 // Terminate the filter options array.
153 filters[filters_count].id = LZMA_VLI_UNKNOWN;
154
155 // If we are using the .lzma format, allow exactly one filter
156 // which has to be LZMA1.
157 if (opt_format == FORMAT_LZMA && (filters_count != 1

--- 5 unchanged lines hidden (view full) ---

163 // filter to prevent LZMA_PROG_ERROR.
164 if (opt_format == FORMAT_XZ)
165 for (size_t i = 0; i < filters_count; ++i)
166 if (filters[i].id == LZMA_FILTER_LZMA1)
167 message_fatal(_("LZMA1 cannot be used "
168 "with the .xz format"));
169
170 // Print the selected filter chain.
148 }
149
150 // Terminate the filter options array.
151 filters[filters_count].id = LZMA_VLI_UNKNOWN;
152
153 // If we are using the .lzma format, allow exactly one filter
154 // which has to be LZMA1.
155 if (opt_format == FORMAT_LZMA && (filters_count != 1

--- 5 unchanged lines hidden (view full) ---

161 // filter to prevent LZMA_PROG_ERROR.
162 if (opt_format == FORMAT_XZ)
163 for (size_t i = 0; i < filters_count; ++i)
164 if (filters[i].id == LZMA_FILTER_LZMA1)
165 message_fatal(_("LZMA1 cannot be used "
166 "with the .xz format"));
167
168 // Print the selected filter chain.
171 message_filters(V_DEBUG, filters);
169 message_filters_show(V_DEBUG, filters);
172
173 // If using --format=raw, we can be decoding. The memusage function
174 // also validates the filter chain and the options used for the
175 // filters.
170
171 // If using --format=raw, we can be decoding. The memusage function
172 // also validates the filter chain and the options used for the
173 // filters.
176 const uint64_t memory_limit = hardware_memlimit_get();
174 const uint64_t memory_limit = hardware_memlimit_get(opt_mode);
177 uint64_t memory_usage;
178 if (opt_mode == MODE_COMPRESS)
179 memory_usage = lzma_raw_encoder_memusage(filters);
180 else
181 memory_usage = lzma_raw_decoder_memusage(filters);
182
183 if (memory_usage == UINT64_MAX)
184 message_fatal(_("Unsupported filter chain or filter options"));
185
186 // Print memory usage info before possible dictionary
187 // size auto-adjusting.
188 message_mem_needed(V_DEBUG, memory_usage);
175 uint64_t memory_usage;
176 if (opt_mode == MODE_COMPRESS)
177 memory_usage = lzma_raw_encoder_memusage(filters);
178 else
179 memory_usage = lzma_raw_decoder_memusage(filters);
180
181 if (memory_usage == UINT64_MAX)
182 message_fatal(_("Unsupported filter chain or filter options"));
183
184 // Print memory usage info before possible dictionary
185 // size auto-adjusting.
186 message_mem_needed(V_DEBUG, memory_usage);
187 if (opt_mode == MODE_COMPRESS) {
188 const uint64_t decmem = lzma_raw_decoder_memusage(filters);
189 if (decmem != UINT64_MAX)
190 message(V_DEBUG, _("Decompression will need "
191 "%s MiB of memory."), uint64_to_str(
192 round_up_to_mib(decmem), 0));
193 }
189
190 if (memory_usage > memory_limit) {
191 // If --no-auto-adjust was used or we didn't find LZMA1 or
192 // LZMA2 as the last filter, give an error immediately.
193 // --format=raw implies --no-auto-adjust.
194
195 if (memory_usage > memory_limit) {
196 // If --no-auto-adjust was used or we didn't find LZMA1 or
197 // LZMA2 as the last filter, give an error immediately.
198 // --format=raw implies --no-auto-adjust.
194 if (!auto_adjust || opt_format == FORMAT_RAW)
199 if (!opt_auto_adjust || opt_format == FORMAT_RAW)
195 memlimit_too_small(memory_usage);
196
197 assert(opt_mode == MODE_COMPRESS);
198
199 // Look for the last filter if it is LZMA2 or LZMA1, so
200 // we can make it use less RAM. With other filters we don't
201 // know what to do.
202 size_t i = 0;

--- 31 unchanged lines hidden (view full) ---

234
235 // Otherwise 1 MiB down and try again. I hope this
236 // isn't too slow method for cases where the original
237 // dict_size is very big.
238 opt->dict_size -= UINT32_C(1) << 20;
239 }
240
241 // Tell the user that we decreased the dictionary size.
200 memlimit_too_small(memory_usage);
201
202 assert(opt_mode == MODE_COMPRESS);
203
204 // Look for the last filter if it is LZMA2 or LZMA1, so
205 // we can make it use less RAM. With other filters we don't
206 // know what to do.
207 size_t i = 0;

--- 31 unchanged lines hidden (view full) ---

239
240 // Otherwise 1 MiB down and try again. I hope this
241 // isn't too slow method for cases where the original
242 // dict_size is very big.
243 opt->dict_size -= UINT32_C(1) << 20;
244 }
245
246 // Tell the user that we decreased the dictionary size.
242 // However, omit the message if no preset or custom chain
243 // was given. FIXME: Always warn?
244 if (!preset_default)
245 message(V_WARNING, _("Adjusted LZMA%c dictionary size "
246 "from %s MiB to %s MiB to not exceed "
247 "the memory usage limit of %s MiB"),
248 filters[i].id == LZMA_FILTER_LZMA2
249 ? '2' : '1',
250 uint64_to_str(orig_dict_size >> 20, 0),
251 uint64_to_str(opt->dict_size >> 20, 1),
252 uint64_to_str(round_up_to_mib(
253 memory_limit), 2));
247 message(V_WARNING, _("Adjusted LZMA%c dictionary size "
248 "from %s MiB to %s MiB to not exceed "
249 "the memory usage limit of %s MiB"),
250 filters[i].id == LZMA_FILTER_LZMA2
251 ? '2' : '1',
252 uint64_to_str(orig_dict_size >> 20, 0),
253 uint64_to_str(opt->dict_size >> 20, 1),
254 uint64_to_str(round_up_to_mib(
255 memory_limit), 2));
254 }
255
256/*
257 // Limit the number of worker threads so that memory usage
258 // limit isn't exceeded.
259 assert(memory_usage > 0);
260 size_t thread_limit = memory_limit / memory_usage;
261 if (thread_limit == 0)

--- 143 unchanged lines hidden (view full) ---

405 && opt_stdout && opt_force)
406 return CODER_INIT_PASSTHRU;
407
408 ret = LZMA_FORMAT_ERROR;
409 break;
410
411 case FORMAT_XZ:
412 ret = lzma_stream_decoder(&strm,
256 }
257
258/*
259 // Limit the number of worker threads so that memory usage
260 // limit isn't exceeded.
261 assert(memory_usage > 0);
262 size_t thread_limit = memory_limit / memory_usage;
263 if (thread_limit == 0)

--- 143 unchanged lines hidden (view full) ---

407 && opt_stdout && opt_force)
408 return CODER_INIT_PASSTHRU;
409
410 ret = LZMA_FORMAT_ERROR;
411 break;
412
413 case FORMAT_XZ:
414 ret = lzma_stream_decoder(&strm,
413 hardware_memlimit_get(), flags);
415 hardware_memlimit_get(
416 MODE_DECOMPRESS), flags);
414 break;
415
416 case FORMAT_LZMA:
417 ret = lzma_alone_decoder(&strm,
417 break;
418
419 case FORMAT_LZMA:
420 ret = lzma_alone_decoder(&strm,
418 hardware_memlimit_get());
421 hardware_memlimit_get(
422 MODE_DECOMPRESS));
419 break;
420
421 case FORMAT_RAW:
422 // Memory usage has already been checked in
423 // coder_set_compression_settings().
424 ret = lzma_raw_decoder(&strm, filters);
425 break;
426 }

--- 233 unchanged lines hidden --- 		423 break;
424
425 case FORMAT_RAW:
426 // Memory usage has already been checked in
427 // coder_set_compression_settings().
428 ret = lzma_raw_decoder(&strm, filters);
429 break;
430 }

--- 233 unchanged lines hidden ---