1/* LTO IL compression streams. 2 3 Copyright (C) 2009-2020 Free Software Foundation, Inc. 4 Contributed by Simon Baldwin <simonb@google.com> 5 6This file is part of GCC. 7 8GCC is free software; you can redistribute it and/or modify it 9under the terms of the GNU General Public License as published by 10the Free Software Foundation; either version 3, or (at your option) 11any later version. 12 13GCC is distributed in the hope that it will be useful, but WITHOUT 14ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 15or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public 16License for more details. 17 18You should have received a copy of the GNU General Public License 19along with GCC; see the file COPYING3. If not see 20<http://www.gnu.org/licenses/>. */ 21 22#include "config.h" 23#include "system.h" 24#include "coretypes.h" 25#include "backend.h" 26#include "tree.h" 27#include "gimple.h" 28#include "cgraph.h" 29#include "lto-streamer.h" 30/* zlib.h includes other system headers. Those headers may test feature 31 test macros. config.h may define feature test macros. For this reason, 32 zlib.h needs to be included after, rather than before, config.h and 33 system.h. */ 34#include <zlib.h> 35#include "lto-compress.h" 36#include "timevar.h" 37 38#ifdef HAVE_ZSTD_H 39#include <zstd.h> 40#endif 41 42/* Compression stream structure, holds the flush callback and opaque token, 43 the buffered data, and a note of whether compressing or uncompressing. */ 44 45struct lto_compression_stream 46{ 47 void (*callback) (const char *, unsigned, void *); 48 void *opaque; 49 char *buffer; 50 size_t bytes; 51 size_t allocation; 52 bool is_compression; 53}; 54 55/* Overall compression constants for zlib. */ 56 57static const size_t Z_BUFFER_LENGTH = 4096; 58static const size_t MIN_STREAM_ALLOCATION = 1024; 59 60/* For zlib, allocate SIZE count of ITEMS and return the address, OPAQUE 61 is unused. */ 62 63static void * 64lto_zalloc (void *opaque, unsigned items, unsigned size) 65{ 66 gcc_assert (opaque == Z_NULL); 67 return xmalloc (items * size); 68} 69 70/* For zlib, free memory at ADDRESS, OPAQUE is unused. */ 71 72static void 73lto_zfree (void *opaque, void *address) 74{ 75 gcc_assert (opaque == Z_NULL); 76 free (address); 77} 78 79/* Return a zlib compression level that zlib will not reject. Normalizes 80 the compression level from the command line flag, clamping non-default 81 values to the appropriate end of their valid range. */ 82 83static int 84lto_normalized_zlib_level (void) 85{ 86 int level = flag_lto_compression_level; 87 88 if (level != Z_DEFAULT_COMPRESSION) 89 { 90 if (level < Z_NO_COMPRESSION) 91 level = Z_NO_COMPRESSION; 92 else if (level > Z_BEST_COMPRESSION) 93 level = Z_BEST_COMPRESSION; 94 } 95 96 return level; 97} 98 99/* Free the buffer and memory associated with STREAM. */ 100 101static void 102lto_destroy_compression_stream (struct lto_compression_stream *stream) 103{ 104 free (stream->buffer); 105 free (stream); 106} 107 108#ifdef HAVE_ZSTD_H 109/* Return a zstd compression level that zstd will not reject. Normalizes 110 the compression level from the command line flag, clamping non-default 111 values to the appropriate end of their valid range. */ 112 113static int 114lto_normalized_zstd_level (void) 115{ 116 int level = flag_lto_compression_level; 117 118 if (level < 0) 119 level = 0; 120 else if (level > ZSTD_maxCLevel ()) 121 level = ZSTD_maxCLevel (); 122 123 return level; 124} 125 126/* Compress STREAM using ZSTD algorithm. */ 127 128static void 129lto_compression_zstd (struct lto_compression_stream *stream) 130{ 131 unsigned char *cursor = (unsigned char *) stream->buffer; 132 size_t size = stream->bytes; 133 134 timevar_push (TV_IPA_LTO_COMPRESS); 135 size_t const outbuf_length = ZSTD_compressBound (size); 136 char *outbuf = (char *) xmalloc (outbuf_length); 137 138 size_t const csize = ZSTD_compress (outbuf, outbuf_length, cursor, size, 139 lto_normalized_zstd_level ()); 140 141 if (ZSTD_isError (csize)) 142 internal_error ("compressed stream: %s", ZSTD_getErrorName (csize)); 143 144 lto_stats.num_compressed_il_bytes += csize; 145 stream->callback (outbuf, csize, NULL); 146 147 lto_destroy_compression_stream (stream); 148 free (outbuf); 149 timevar_pop (TV_IPA_LTO_COMPRESS); 150} 151 152/* Uncompress STREAM using ZSTD algorithm. */ 153 154static void 155lto_uncompression_zstd (struct lto_compression_stream *stream) 156{ 157 unsigned char *cursor = (unsigned char *) stream->buffer; 158 size_t size = stream->bytes; 159 160 timevar_push (TV_IPA_LTO_DECOMPRESS); 161 unsigned long long const rsize = ZSTD_getFrameContentSize (cursor, size); 162 if (rsize == ZSTD_CONTENTSIZE_ERROR) 163 internal_error ("original not compressed with zstd"); 164 else if (rsize == ZSTD_CONTENTSIZE_UNKNOWN) 165 internal_error ("original size unknown"); 166 167 char *outbuf = (char *) xmalloc (rsize); 168 size_t const dsize = ZSTD_decompress (outbuf, rsize, cursor, size); 169 170 if (ZSTD_isError (dsize)) 171 internal_error ("decompressed stream: %s", ZSTD_getErrorName (dsize)); 172 173 lto_stats.num_uncompressed_il_bytes += dsize; 174 stream->callback (outbuf, dsize, stream->opaque); 175 176 lto_destroy_compression_stream (stream); 177 free (outbuf); 178 timevar_pop (TV_IPA_LTO_DECOMPRESS); 179} 180 181#endif 182 183/* Create a new compression stream, with CALLBACK flush function passed 184 OPAQUE token, IS_COMPRESSION indicates if compressing or uncompressing. */ 185 186static struct lto_compression_stream * 187lto_new_compression_stream (void (*callback) (const char *, unsigned, void *), 188 void *opaque, bool is_compression) 189{ 190 struct lto_compression_stream *stream 191 = (struct lto_compression_stream *) xmalloc (sizeof (*stream)); 192 193 memset (stream, 0, sizeof (*stream)); 194 stream->callback = callback; 195 stream->opaque = opaque; 196 stream->is_compression = is_compression; 197 198 return stream; 199} 200 201/* Append NUM_CHARS from address BASE to STREAM. */ 202 203static void 204lto_append_to_compression_stream (struct lto_compression_stream *stream, 205 const char *base, size_t num_chars) 206{ 207 size_t required = stream->bytes + num_chars; 208 209 if (stream->allocation < required) 210 { 211 if (stream->allocation == 0) 212 stream->allocation = MIN_STREAM_ALLOCATION; 213 while (stream->allocation < required) 214 stream->allocation *= 2; 215 216 stream->buffer = (char *) xrealloc (stream->buffer, stream->allocation); 217 } 218 219 memcpy (stream->buffer + stream->bytes, base, num_chars); 220 stream->bytes += num_chars; 221} 222 223/* Return a new compression stream, with CALLBACK flush function passed 224 OPAQUE token. */ 225 226struct lto_compression_stream * 227lto_start_compression (void (*callback) (const char *, unsigned, void *), 228 void *opaque) 229{ 230 return lto_new_compression_stream (callback, opaque, true); 231} 232 233/* Append NUM_CHARS from address BASE to STREAM. */ 234 235void 236lto_compress_block (struct lto_compression_stream *stream, 237 const char *base, size_t num_chars) 238{ 239 gcc_assert (stream->is_compression); 240 241 lto_append_to_compression_stream (stream, base, num_chars); 242 lto_stats.num_output_il_bytes += num_chars; 243} 244 245static void ATTRIBUTE_UNUSED 246lto_compression_zlib (struct lto_compression_stream *stream) 247{ 248 unsigned char *cursor = (unsigned char *) stream->buffer; 249 size_t remaining = stream->bytes; 250 const size_t outbuf_length = Z_BUFFER_LENGTH; 251 unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length); 252 z_stream out_stream; 253 size_t compressed_bytes = 0; 254 int status; 255 256 gcc_assert (stream->is_compression); 257 258 timevar_push (TV_IPA_LTO_COMPRESS); 259 260 out_stream.next_out = outbuf; 261 out_stream.avail_out = outbuf_length; 262 out_stream.next_in = cursor; 263 out_stream.avail_in = remaining; 264 out_stream.zalloc = lto_zalloc; 265 out_stream.zfree = lto_zfree; 266 out_stream.opaque = Z_NULL; 267 268 status = deflateInit (&out_stream, lto_normalized_zlib_level ()); 269 if (status != Z_OK) 270 internal_error ("compressed stream: %s", zError (status)); 271 272 do 273 { 274 size_t in_bytes, out_bytes; 275 276 status = deflate (&out_stream, Z_FINISH); 277 if (status != Z_OK && status != Z_STREAM_END) 278 internal_error ("compressed stream: %s", zError (status)); 279 280 in_bytes = remaining - out_stream.avail_in; 281 out_bytes = outbuf_length - out_stream.avail_out; 282 283 stream->callback ((const char *) outbuf, out_bytes, stream->opaque); 284 lto_stats.num_compressed_il_bytes += out_bytes; 285 compressed_bytes += out_bytes; 286 287 cursor += in_bytes; 288 remaining -= in_bytes; 289 290 out_stream.next_out = outbuf; 291 out_stream.avail_out = outbuf_length; 292 out_stream.next_in = cursor; 293 out_stream.avail_in = remaining; 294 } 295 while (status != Z_STREAM_END); 296 297 status = deflateEnd (&out_stream); 298 if (status != Z_OK) 299 internal_error ("compressed stream: %s", zError (status)); 300 301 lto_destroy_compression_stream (stream); 302 free (outbuf); 303 timevar_pop (TV_IPA_LTO_COMPRESS); 304} 305 306void 307lto_end_compression (struct lto_compression_stream *stream) 308{ 309#ifdef HAVE_ZSTD_H 310 lto_compression_zstd (stream); 311#else 312 lto_compression_zlib (stream); 313#endif 314} 315 316/* Return a new uncompression stream, with CALLBACK flush function passed 317 OPAQUE token. */ 318 319struct lto_compression_stream * 320lto_start_uncompression (void (*callback) (const char *, unsigned, void *), 321 void *opaque) 322{ 323 return lto_new_compression_stream (callback, opaque, false); 324} 325 326/* Append NUM_CHARS from address BASE to STREAM. */ 327 328void 329lto_uncompress_block (struct lto_compression_stream *stream, 330 const char *base, size_t num_chars) 331{ 332 gcc_assert (!stream->is_compression); 333 334 lto_append_to_compression_stream (stream, base, num_chars); 335 lto_stats.num_input_il_bytes += num_chars; 336} 337 338static void 339lto_uncompression_zlib (struct lto_compression_stream *stream) 340{ 341 unsigned char *cursor = (unsigned char *) stream->buffer; 342 size_t remaining = stream->bytes; 343 const size_t outbuf_length = Z_BUFFER_LENGTH; 344 unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length); 345 size_t uncompressed_bytes = 0; 346 347 gcc_assert (!stream->is_compression); 348 timevar_push (TV_IPA_LTO_DECOMPRESS); 349 350 while (remaining > 0) 351 { 352 z_stream in_stream; 353 size_t out_bytes; 354 int status; 355 356 in_stream.next_out = outbuf; 357 in_stream.avail_out = outbuf_length; 358 in_stream.next_in = cursor; 359 in_stream.avail_in = remaining; 360 in_stream.zalloc = lto_zalloc; 361 in_stream.zfree = lto_zfree; 362 in_stream.opaque = Z_NULL; 363 364 status = inflateInit (&in_stream); 365 if (status != Z_OK) 366 internal_error ("compressed stream: %s", zError (status)); 367 368 do 369 { 370 size_t in_bytes; 371 372 status = inflate (&in_stream, Z_SYNC_FLUSH); 373 if (status != Z_OK && status != Z_STREAM_END) 374 internal_error ("compressed stream: %s", zError (status)); 375 376 in_bytes = remaining - in_stream.avail_in; 377 out_bytes = outbuf_length - in_stream.avail_out; 378 379 stream->callback ((const char *) outbuf, out_bytes, stream->opaque); 380 lto_stats.num_uncompressed_il_bytes += out_bytes; 381 uncompressed_bytes += out_bytes; 382 383 cursor += in_bytes; 384 remaining -= in_bytes; 385 386 in_stream.next_out = outbuf; 387 in_stream.avail_out = outbuf_length; 388 in_stream.next_in = cursor; 389 in_stream.avail_in = remaining; 390 } 391 while (!(status == Z_STREAM_END && out_bytes == 0)); 392 393 status = inflateEnd (&in_stream); 394 if (status != Z_OK) 395 internal_error ("compressed stream: %s", zError (status)); 396 } 397 398 lto_destroy_compression_stream (stream); 399 free (outbuf); 400 timevar_pop (TV_IPA_LTO_DECOMPRESS); 401} 402 403void 404lto_end_uncompression (struct lto_compression_stream *stream, 405 lto_compression compression) 406{ 407#ifdef HAVE_ZSTD_H 408 if (compression == ZSTD) 409 { 410 lto_uncompression_zstd (stream); 411 return; 412 } 413#endif 414 if (compression == ZSTD) 415 internal_error ("compiler does not support ZSTD LTO compression"); 416 417 lto_uncompression_zlib (stream); 418} 419