1/////////////////////////////////////////////////////////////////////////////// 2// 3/// \file 04_compress_easy_mt.c 4/// \brief Compress in multi-call mode using LZMA2 in multi-threaded mode 5/// 6/// Usage: ./04_compress_easy_mt < INFILE > OUTFILE 7/// 8/// Example: ./04_compress_easy_mt < foo > foo.xz 9// 10// Author: Lasse Collin 11// 12// This file has been put into the public domain. 13// You can do whatever you want with this file. 14// 15/////////////////////////////////////////////////////////////////////////////// 16 17#include <stdbool.h> 18#include <stdlib.h> 19#include <stdio.h> 20#include <string.h> 21#include <errno.h> 22#include <lzma.h> 23 24 25static bool 26init_encoder(lzma_stream *strm) 27{ 28 // The threaded encoder takes the options as pointer to 29 // a lzma_mt structure. 30 lzma_mt mt = { 31 // No flags are needed. 32 .flags = 0, 33 34 // Let liblzma determine a sane block size. 35 .block_size = 0, 36 37 // Use no timeout for lzma_code() calls by setting timeout 38 // to zero. That is, sometimes lzma_code() might block for 39 // a long time (from several seconds to even minutes). 40 // If this is not OK, for example due to progress indicator 41 // needing updates, specify a timeout in milliseconds here. 42 // See the documentation of lzma_mt in lzma/container.h for 43 // information how to choose a reasonable timeout. 44 .timeout = 0, 45 46 // Use the default preset (6) for LZMA2. 47 // To use a preset, filters must be set to NULL. 48 .preset = LZMA_PRESET_DEFAULT, 49 .filters = NULL, 50 51 // Use CRC64 for integrity checking. See also 52 // 01_compress_easy.c about choosing the integrity check. 53 .check = LZMA_CHECK_CRC64, 54 }; 55 56 // Detect how many threads the CPU supports. 57 mt.threads = lzma_cputhreads(); 58 59 // If the number of CPU cores/threads cannot be detected, 60 // use one thread. Note that this isn't the same as the normal 61 // single-threaded mode as this will still split the data into 62 // blocks and use more RAM than the normal single-threaded mode. 63 // You may want to consider using lzma_easy_encoder() or 64 // lzma_stream_encoder() instead of lzma_stream_encoder_mt() if 65 // lzma_cputhreads() returns 0 or 1. 66 if (mt.threads == 0) 67 mt.threads = 1; 68 69 // If the number of CPU cores/threads exceeds threads_max, 70 // limit the number of threads to keep memory usage lower. 71 // The number 8 is arbitrarily chosen and may be too low or 72 // high depending on the compression preset and the computer 73 // being used. 74 // 75 // FIXME: A better way could be to check the amount of RAM 76 // (or available RAM) and use lzma_stream_encoder_mt_memusage() 77 // to determine if the number of threads should be reduced. 78 const uint32_t threads_max = 8; 79 if (mt.threads > threads_max) 80 mt.threads = threads_max; 81 82 // Initialize the threaded encoder. 83 lzma_ret ret = lzma_stream_encoder_mt(strm, &mt); 84 85 if (ret == LZMA_OK) 86 return true; 87 88 const char *msg; 89 switch (ret) { 90 case LZMA_MEM_ERROR: 91 msg = "Memory allocation failed"; 92 break; 93 94 case LZMA_OPTIONS_ERROR: 95 // We are no longer using a plain preset so this error 96 // message has been edited accordingly compared to 97 // 01_compress_easy.c. 98 msg = "Specified filter chain is not supported"; 99 break; 100 101 case LZMA_UNSUPPORTED_CHECK: 102 msg = "Specified integrity check is not supported"; 103 break; 104 105 default: 106 msg = "Unknown error, possibly a bug"; 107 break; 108 } 109 110 fprintf(stderr, "Error initializing the encoder: %s (error code %u)\n", 111 msg, ret); 112 return false; 113} 114 115 116// This function is identical to the one in 01_compress_easy.c. 117static bool 118compress(lzma_stream *strm, FILE *infile, FILE *outfile) 119{ 120 lzma_action action = LZMA_RUN; 121 122 uint8_t inbuf[BUFSIZ]; 123 uint8_t outbuf[BUFSIZ]; 124 125 strm->next_in = NULL; 126 strm->avail_in = 0; 127 strm->next_out = outbuf; 128 strm->avail_out = sizeof(outbuf); 129 130 while (true) { 131 if (strm->avail_in == 0 && !feof(infile)) { 132 strm->next_in = inbuf; 133 strm->avail_in = fread(inbuf, 1, sizeof(inbuf), 134 infile); 135 136 if (ferror(infile)) { 137 fprintf(stderr, "Read error: %s\n", 138 strerror(errno)); 139 return false; 140 } 141 142 if (feof(infile)) 143 action = LZMA_FINISH; 144 } 145 146 lzma_ret ret = lzma_code(strm, action); 147 148 if (strm->avail_out == 0 || ret == LZMA_STREAM_END) { 149 size_t write_size = sizeof(outbuf) - strm->avail_out; 150 151 if (fwrite(outbuf, 1, write_size, outfile) 152 != write_size) { 153 fprintf(stderr, "Write error: %s\n", 154 strerror(errno)); 155 return false; 156 } 157 158 strm->next_out = outbuf; 159 strm->avail_out = sizeof(outbuf); 160 } 161 162 if (ret != LZMA_OK) { 163 if (ret == LZMA_STREAM_END) 164 return true; 165 166 const char *msg; 167 switch (ret) { 168 case LZMA_MEM_ERROR: 169 msg = "Memory allocation failed"; 170 break; 171 172 case LZMA_DATA_ERROR: 173 msg = "File size limits exceeded"; 174 break; 175 176 default: 177 msg = "Unknown error, possibly a bug"; 178 break; 179 } 180 181 fprintf(stderr, "Encoder error: %s (error code %u)\n", 182 msg, ret); 183 return false; 184 } 185 } 186} 187 188 189extern int 190main(void) 191{ 192 lzma_stream strm = LZMA_STREAM_INIT; 193 194 bool success = init_encoder(&strm); 195 if (success) 196 success = compress(&strm, stdin, stdout); 197 198 lzma_end(&strm); 199 200 if (fclose(stdout)) { 201 fprintf(stderr, "Write error: %s\n", strerror(errno)); 202 success = false; 203 } 204 205 return success ? EXIT_SUCCESS : EXIT_FAILURE; 206} 207