1// SPDX-License-Identifier: 0BSD 2 3/////////////////////////////////////////////////////////////////////////////// 4// 5/// \file hardware.c 6/// \brief Detection of available hardware resources 7// 8// Author: Lasse Collin 9// 10/////////////////////////////////////////////////////////////////////////////// 11 12#include "private.h" 13 14 15/// Maximum number of worker threads. This can be set with 16/// the --threads=NUM command line option. 17static uint32_t threads_max; 18 19/// True when the number of threads is automatically determined based 20/// on the available hardware threads. 21static bool threads_are_automatic = false; 22 23/// If true, then try to use multi-threaded mode (if memlimit allows) 24/// even if only one thread was requested explicitly (-T+1). 25static bool use_mt_mode_with_one_thread = false; 26 27/// Memory usage limit for compression 28static uint64_t memlimit_compress = 0; 29 30/// Memory usage limit for decompression 31static uint64_t memlimit_decompress = 0; 32 33/// Default memory usage for multithreaded modes: 34/// 35/// - Default value for --memlimit-compress when automatic number of threads 36/// is used. However, if the limit wouldn't allow even one thread then 37/// the limit is ignored in coder.c and one thread will be used anyway. 38/// This mess is a compromise: we wish to prevent -T0 from using too 39/// many threads but we also don't want xz to give an error due to 40/// a memlimit that the user didn't explicitly set. 41/// 42/// - Default value for --memlimit-mt-decompress 43/// 44/// This value is calculated in hardware_init() and cannot be changed later. 45static uint64_t memlimit_mt_default; 46 47/// Memory usage limit for multithreaded decompression. This is a soft limit: 48/// if reducing the number of threads to one isn't enough to keep memory 49/// usage below this limit, then one thread is used and this limit is ignored. 50/// memlimit_decompress is still obeyed. 51/// 52/// This can be set with --memlimit-mt-decompress. The default value for 53/// this is memlimit_mt_default. 54static uint64_t memlimit_mtdec; 55 56/// Total amount of physical RAM 57static uint64_t total_ram; 58 59 60extern void 61hardware_threads_set(uint32_t n) 62{ 63 // Reset these to false first and set them to true when appropriate. 64 threads_are_automatic = false; 65 use_mt_mode_with_one_thread = false; 66 67 if (n == 0) { 68 // Automatic number of threads was requested. 69 // If there is only one hardware thread, multi-threaded 70 // mode will still be used if memory limit allows. 71 threads_are_automatic = true; 72 use_mt_mode_with_one_thread = true; 73 74 // If threading support was enabled at build time, 75 // use the number of available CPU cores. Otherwise 76 // use one thread since disabling threading support 77 // omits lzma_cputhreads() from liblzma. 78#ifdef MYTHREAD_ENABLED 79 threads_max = lzma_cputhreads(); 80 if (threads_max == 0) 81 threads_max = 1; 82#else 83 threads_max = 1; 84#endif 85 } else if (n == UINT32_MAX) { 86 use_mt_mode_with_one_thread = true; 87 threads_max = 1; 88 } else { 89 threads_max = n; 90 } 91 92 return; 93} 94 95 96extern uint32_t 97hardware_threads_get(void) 98{ 99 return threads_max; 100} 101 102 103extern bool 104hardware_threads_is_mt(void) 105{ 106#ifdef MYTHREAD_ENABLED 107 return threads_max > 1 || use_mt_mode_with_one_thread; 108#else 109 return false; 110#endif 111} 112 113 114extern void 115hardware_memlimit_set(uint64_t new_memlimit, 116 bool set_compress, bool set_decompress, bool set_mtdec, 117 bool is_percentage) 118{ 119 if (is_percentage) { 120 assert(new_memlimit > 0); 121 assert(new_memlimit <= 100); 122 new_memlimit = (uint32_t)new_memlimit * total_ram / 100; 123 } 124 125 if (set_compress) { 126 memlimit_compress = new_memlimit; 127 128#if SIZE_MAX == UINT32_MAX 129 // FIXME? 130 // 131 // When running a 32-bit xz on a system with a lot of RAM and 132 // using a percentage-based memory limit, the result can be 133 // bigger than the 32-bit address space. Limiting the limit 134 // below SIZE_MAX for compression (not decompression) makes 135 // xz lower the compression settings (or number of threads) 136 // to a level that *might* work. In practice it has worked 137 // when using a 64-bit kernel that gives full 4 GiB address 138 // space to 32-bit programs. In other situations this might 139 // still be too high, like 32-bit kernels that may give much 140 // less than 4 GiB to a single application. 141 // 142 // So this is an ugly hack but I will keep it here while 143 // it does more good than bad. 144 // 145 // Use a value less than SIZE_MAX so that there's some room 146 // for the xz program and so on. Don't use 4000 MiB because 147 // it could look like someone mixed up base-2 and base-10. 148#ifdef __mips__ 149 // For MIPS32, due to architectural peculiarities, 150 // the limit is even lower. 151 const uint64_t limit_max = UINT64_C(2000) << 20; 152#else 153 const uint64_t limit_max = UINT64_C(4020) << 20; 154#endif 155 156 // UINT64_MAX is a special case for the string "max" so 157 // that has to be handled specially. 158 if (memlimit_compress != UINT64_MAX 159 && memlimit_compress > limit_max) 160 memlimit_compress = limit_max; 161#endif 162 } 163 164 if (set_decompress) 165 memlimit_decompress = new_memlimit; 166 167 if (set_mtdec) 168 memlimit_mtdec = new_memlimit; 169 170 return; 171} 172 173 174extern uint64_t 175hardware_memlimit_get(enum operation_mode mode) 176{ 177 // 0 is a special value that indicates the default. 178 // It disables the limit in single-threaded mode. 179 // 180 // NOTE: For multithreaded decompression, this is the hard limit 181 // (memlimit_stop). hardware_memlimit_mtdec_get() gives the 182 // soft limit (memlimit_threaded). 183 const uint64_t memlimit = mode == MODE_COMPRESS 184 ? memlimit_compress : memlimit_decompress; 185 return memlimit != 0 ? memlimit : UINT64_MAX; 186} 187 188 189extern uint64_t 190hardware_memlimit_mtenc_get(void) 191{ 192 return hardware_memlimit_mtenc_is_default() 193 ? memlimit_mt_default 194 : hardware_memlimit_get(MODE_COMPRESS); 195} 196 197 198extern bool 199hardware_memlimit_mtenc_is_default(void) 200{ 201 return memlimit_compress == 0 && threads_are_automatic; 202} 203 204 205extern uint64_t 206hardware_memlimit_mtdec_get(void) 207{ 208 uint64_t m = memlimit_mtdec != 0 209 ? memlimit_mtdec 210 : memlimit_mt_default; 211 212 // Cap the value to memlimit_decompress if it has been specified. 213 // This is nice for --info-memory. It wouldn't be needed for liblzma 214 // since it does this anyway. 215 if (memlimit_decompress != 0 && m > memlimit_decompress) 216 m = memlimit_decompress; 217 218 return m; 219} 220 221 222/// Helper for hardware_memlimit_show() to print one human-readable info line. 223static void 224memlimit_show(const char *str, size_t str_columns, uint64_t value) 225{ 226 // Calculate the field width so that str will be padded to take 227 // str_columns on the terminal. 228 // 229 // NOTE: If the string is invalid, this will be -1. Using -1 as 230 // the field width is fine here so it's not handled specially. 231 const int fw = tuklib_mbstr_fw(str, (int)(str_columns)); 232 233 // The memory usage limit is considered to be disabled if value 234 // is 0 or UINT64_MAX. This might get a bit more complex once there 235 // is threading support. See the comment in hardware_memlimit_get(). 236 if (value == 0 || value == UINT64_MAX) 237 printf(" %-*s %s\n", fw, str, _("Disabled")); 238 else 239 printf(" %-*s %s MiB (%s B)\n", fw, str, 240 uint64_to_str(round_up_to_mib(value), 0), 241 uint64_to_str(value, 1)); 242 243 return; 244} 245 246 247extern void 248hardware_memlimit_show(void) 249{ 250 uint32_t cputhreads = 1; 251#ifdef MYTHREAD_ENABLED 252 cputhreads = lzma_cputhreads(); 253 if (cputhreads == 0) 254 cputhreads = 1; 255#endif 256 257 if (opt_robot) { 258 printf("%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 259 "\t%" PRIu64 "\t%" PRIu32 "\n", 260 total_ram, 261 memlimit_compress, 262 memlimit_decompress, 263 hardware_memlimit_mtdec_get(), 264 memlimit_mt_default, 265 cputhreads); 266 } else { 267 const char *msgs[] = { 268 _("Amount of physical memory (RAM):"), 269 _("Number of processor threads:"), 270 _("Compression:"), 271 _("Decompression:"), 272 _("Multi-threaded decompression:"), 273 _("Default for -T0:"), 274 }; 275 276 size_t width_max = 1; 277 for (unsigned i = 0; i < ARRAY_SIZE(msgs); ++i) { 278 size_t w = tuklib_mbstr_width(msgs[i], NULL); 279 280 // When debugging, catch invalid strings with 281 // an assertion. Otherwise fallback to 1 so 282 // that the columns just won't be aligned. 283 assert(w != (size_t)-1); 284 if (w == (size_t)-1) 285 w = 1; 286 287 if (width_max < w) 288 width_max = w; 289 } 290 291 puts(_("Hardware information:")); 292 memlimit_show(msgs[0], width_max, total_ram); 293 printf(" %-*s %" PRIu32 "\n", 294 tuklib_mbstr_fw(msgs[1], (int)(width_max)), 295 msgs[1], cputhreads); 296 297 putchar('\n'); 298 puts(_("Memory usage limits:")); 299 memlimit_show(msgs[2], width_max, memlimit_compress); 300 memlimit_show(msgs[3], width_max, memlimit_decompress); 301 memlimit_show(msgs[4], width_max, 302 hardware_memlimit_mtdec_get()); 303 memlimit_show(msgs[5], width_max, memlimit_mt_default); 304 } 305 306 tuklib_exit(E_SUCCESS, E_ERROR, message_verbosity_get() != V_SILENT); 307} 308 309 310extern void 311hardware_init(void) 312{ 313 // Get the amount of RAM. If we cannot determine it, 314 // use the assumption defined by the configure script. 315 total_ram = lzma_physmem(); 316 if (total_ram == 0) 317 total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024; 318 319 // FIXME? There may be better methods to determine the default value. 320 // One Linux-specific suggestion is to use MemAvailable from 321 // /proc/meminfo as the starting point. 322 memlimit_mt_default = total_ram / 4; 323 324#if SIZE_MAX == UINT32_MAX 325 // A too high value may cause 32-bit xz to run out of address space. 326 // Use a conservative maximum value here. A few typical address space 327 // sizes with Linux: 328 // - x86-64 with 32-bit xz: 4 GiB 329 // - x86: 3 GiB 330 // - MIPS32: 2 GiB 331 const size_t mem_ceiling = 1400U << 20; 332 if (memlimit_mt_default > mem_ceiling) 333 memlimit_mt_default = mem_ceiling; 334#endif 335 336 // Enable threaded mode by default. xz 5.4.x and older 337 // used single-threaded mode by default. 338 hardware_threads_set(0); 339 340 return; 341} 342