1/* 2LZ4 auto-framing library 3Copyright (C) 2011-2015, Yann Collet. 4 5BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 7Redistribution and use in source and binary forms, with or without 8modification, are permitted provided that the following conditions are 9met: 10 11* Redistributions of source code must retain the above copyright 12notice, this list of conditions and the following disclaimer. 13* Redistributions in binary form must reproduce the above 14copyright notice, this list of conditions and the following disclaimer 15in the documentation and/or other materials provided with the 16distribution. 17 18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30You can contact the author at : 31- LZ4 source repository : https://github.com/Cyan4973/lz4 32- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 33*/ 34 35/* LZ4F is a stand-alone API to create LZ4-compressed Frames 36* in full conformance with specification v1.5.0 37* All related operations, including memory management, are handled by the library. 38* */ 39 40 41/************************************** 42* Compiler Options 43**************************************/ 44#ifdef _MSC_VER /* Visual Studio */ 45# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 46#endif 47 48 49/************************************** 50* Memory routines 51**************************************/ 52#include <stdlib.h> /* malloc, calloc, free */ 53#define ALLOCATOR(s) calloc(1,s) 54#define FREEMEM free 55#include <string.h> /* memset, memcpy, memmove */ 56#define MEM_INIT memset 57 58 59/************************************** 60* Includes 61**************************************/ 62#include "lz4frame_static.h" 63#include "lz4.h" 64#include "lz4hc.h" 65#include "xxhash.h" 66 67 68/************************************** 69* Basic Types 70**************************************/ 71#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ 72# include <stdint.h> 73typedef uint8_t BYTE; 74typedef uint16_t U16; 75typedef uint32_t U32; 76typedef int32_t S32; 77typedef uint64_t U64; 78#else 79typedef unsigned char BYTE; 80typedef unsigned short U16; 81typedef unsigned int U32; 82typedef signed int S32; 83typedef unsigned long long U64; 84#endif 85 86 87/************************************** 88* Constants 89**************************************/ 90#define KB *(1<<10) 91#define MB *(1<<20) 92#define GB *(1<<30) 93 94#define _1BIT 0x01 95#define _2BITS 0x03 96#define _3BITS 0x07 97#define _4BITS 0x0F 98#define _8BITS 0xFF 99 100#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U 101#define LZ4F_MAGICNUMBER 0x184D2204U 102#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U 103#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB 104 105static const size_t minFHSize = 7; 106static const size_t maxFHSize = 15; 107static const size_t BHSize = 4; 108static const int minHClevel = 3; 109 110 111/************************************** 112* Structures and local types 113**************************************/ 114typedef struct LZ4F_cctx_s 115{ 116 LZ4F_preferences_t prefs; 117 U32 version; 118 U32 cStage; 119 size_t maxBlockSize; 120 size_t maxBufferSize; 121 BYTE* tmpBuff; 122 BYTE* tmpIn; 123 size_t tmpInSize; 124 U64 totalInSize; 125 XXH32_state_t xxh; 126 void* lz4CtxPtr; 127 U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ 128} LZ4F_cctx_t; 129 130typedef struct LZ4F_dctx_s 131{ 132 LZ4F_frameInfo_t frameInfo; 133 U32 version; 134 U32 dStage; 135 U64 frameRemainingSize; 136 size_t maxBlockSize; 137 size_t maxBufferSize; 138 const BYTE* srcExpect; 139 BYTE* tmpIn; 140 size_t tmpInSize; 141 size_t tmpInTarget; 142 BYTE* tmpOutBuffer; 143 const BYTE* dict; 144 size_t dictSize; 145 BYTE* tmpOut; 146 size_t tmpOutSize; 147 size_t tmpOutStart; 148 XXH32_state_t xxh; 149 BYTE header[16]; 150} LZ4F_dctx_t; 151 152 153/************************************** 154* Error management 155**************************************/ 156#define LZ4F_GENERATE_STRING(STRING) #STRING, 157static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; 158 159 160unsigned LZ4F_isError(LZ4F_errorCode_t code) 161{ 162 return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode)); 163} 164 165const char* LZ4F_getErrorName(LZ4F_errorCode_t code) 166{ 167 static const char* codeError = "Unspecified error code"; 168 if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)]; 169 return codeError; 170} 171 172 173/************************************** 174* Private functions 175**************************************/ 176static size_t LZ4F_getBlockSize(unsigned blockSizeID) 177{ 178 static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; 179 180 if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; 181 blockSizeID -= 4; 182 if (blockSizeID > 3) return (size_t)-LZ4F_ERROR_maxBlockSize_invalid; 183 return blockSizes[blockSizeID]; 184} 185 186 187/* unoptimized version; solves endianess & alignment issues */ 188static U32 LZ4F_readLE32 (const BYTE* srcPtr) 189{ 190 U32 value32 = srcPtr[0]; 191 value32 += (srcPtr[1]<<8); 192 value32 += (srcPtr[2]<<16); 193 value32 += ((U32)srcPtr[3])<<24; 194 return value32; 195} 196 197static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32) 198{ 199 dstPtr[0] = (BYTE)value32; 200 dstPtr[1] = (BYTE)(value32 >> 8); 201 dstPtr[2] = (BYTE)(value32 >> 16); 202 dstPtr[3] = (BYTE)(value32 >> 24); 203} 204 205static U64 LZ4F_readLE64 (const BYTE* srcPtr) 206{ 207 U64 value64 = srcPtr[0]; 208 value64 += ((U64)srcPtr[1]<<8); 209 value64 += ((U64)srcPtr[2]<<16); 210 value64 += ((U64)srcPtr[3]<<24); 211 value64 += ((U64)srcPtr[4]<<32); 212 value64 += ((U64)srcPtr[5]<<40); 213 value64 += ((U64)srcPtr[6]<<48); 214 value64 += ((U64)srcPtr[7]<<56); 215 return value64; 216} 217 218static void LZ4F_writeLE64 (BYTE* dstPtr, U64 value64) 219{ 220 dstPtr[0] = (BYTE)value64; 221 dstPtr[1] = (BYTE)(value64 >> 8); 222 dstPtr[2] = (BYTE)(value64 >> 16); 223 dstPtr[3] = (BYTE)(value64 >> 24); 224 dstPtr[4] = (BYTE)(value64 >> 32); 225 dstPtr[5] = (BYTE)(value64 >> 40); 226 dstPtr[6] = (BYTE)(value64 >> 48); 227 dstPtr[7] = (BYTE)(value64 >> 56); 228} 229 230 231static BYTE LZ4F_headerChecksum (const void* header, size_t length) 232{ 233 U32 xxh = XXH32(header, length, 0); 234 return (BYTE)(xxh >> 8); 235} 236 237 238/************************************** 239* Simple compression functions 240**************************************/ 241static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize) 242{ 243 LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; 244 size_t maxBlockSize = 64 KB; 245 while (requestedBSID > proposedBSID) 246 { 247 if (srcSize <= maxBlockSize) 248 return proposedBSID; 249 proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1); 250 maxBlockSize <<= 2; 251 } 252 return requestedBSID; 253} 254 255 256size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) 257{ 258 LZ4F_preferences_t prefs; 259 size_t headerSize; 260 size_t streamSize; 261 262 if (preferencesPtr!=NULL) prefs = *preferencesPtr; 263 else memset(&prefs, 0, sizeof(prefs)); 264 265 prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); 266 prefs.autoFlush = 1; 267 268 headerSize = maxFHSize; /* header size, including magic number and frame content size*/ 269 streamSize = LZ4F_compressBound(srcSize, &prefs); 270 271 return headerSize + streamSize; 272} 273 274 275/* LZ4F_compressFrame() 276* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step. 277* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. 278* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() 279* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) 280* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. 281* The result of the function is the number of bytes written into dstBuffer. 282* The function outputs an error code if it fails (can be tested using LZ4F_isError()) 283*/ 284size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) 285{ 286 LZ4F_cctx_t cctxI; 287 LZ4_stream_t lz4ctx; 288 LZ4F_preferences_t prefs; 289 LZ4F_compressOptions_t options; 290 LZ4F_errorCode_t errorCode; 291 BYTE* const dstStart = (BYTE*) dstBuffer; 292 BYTE* dstPtr = dstStart; 293 BYTE* const dstEnd = dstStart + dstMaxSize; 294 295 memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ 296 memset(&options, 0, sizeof(options)); 297 298 cctxI.version = LZ4F_VERSION; 299 cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ 300 301 if (preferencesPtr!=NULL) 302 prefs = *preferencesPtr; 303 else 304 memset(&prefs, 0, sizeof(prefs)); 305 if (prefs.frameInfo.contentSize != 0) 306 prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ 307 308 if (prefs.compressionLevel < (int)minHClevel) 309 { 310 cctxI.lz4CtxPtr = &lz4ctx; 311 cctxI.lz4CtxLevel = 1; 312 } 313 314 prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); 315 prefs.autoFlush = 1; 316 if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) 317 prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* no need for linked blocks */ 318 319 options.stableSrc = 1; 320 321 if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs)) 322 return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; 323 324 errorCode = LZ4F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ 325 if (LZ4F_isError(errorCode)) return errorCode; 326 dstPtr += errorCode; /* header size */ 327 328 errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); 329 if (LZ4F_isError(errorCode)) return errorCode; 330 dstPtr += errorCode; 331 332 errorCode = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ 333 if (LZ4F_isError(errorCode)) return errorCode; 334 dstPtr += errorCode; 335 336 if (prefs.compressionLevel >= (int)minHClevel) /* no allocation necessary with lz4 fast */ 337 FREEMEM(cctxI.lz4CtxPtr); 338 339 return (dstPtr - dstStart); 340} 341 342 343/*********************************** 344* Advanced compression functions 345***********************************/ 346 347/* LZ4F_createCompressionContext() : 348* The first thing to do is to create a compressionContext object, which will be used in all compression operations. 349* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. 350* The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. 351* The function will provide a pointer to an allocated LZ4F_compressionContext_t object. 352* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. 353* Object can release its memory using LZ4F_freeCompressionContext(); 354*/ 355LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) 356{ 357 LZ4F_cctx_t* cctxPtr; 358 359 cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t)); 360 if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-LZ4F_ERROR_allocation_failed); 361 362 cctxPtr->version = version; 363 cctxPtr->cStage = 0; /* Next stage : write header */ 364 365 *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; 366 367 return LZ4F_OK_NoError; 368} 369 370 371LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) 372{ 373 LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; 374 375 if (cctxPtr != NULL) /* null pointers can be safely provided to this function, like free() */ 376 { 377 FREEMEM(cctxPtr->lz4CtxPtr); 378 FREEMEM(cctxPtr->tmpBuff); 379 FREEMEM(LZ4F_compressionContext); 380 } 381 382 return LZ4F_OK_NoError; 383} 384 385 386/* LZ4F_compressBegin() : 387* will write the frame header into dstBuffer. 388* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. 389* The result of the function is the number of bytes written into dstBuffer for the header 390* or an error code (can be tested using LZ4F_isError()) 391*/ 392size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) 393{ 394 LZ4F_preferences_t prefNull; 395 LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; 396 BYTE* const dstStart = (BYTE*)dstBuffer; 397 BYTE* dstPtr = dstStart; 398 BYTE* headerStart; 399 size_t requiredBuffSize; 400 401 if (dstMaxSize < maxFHSize) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; 402 if (cctxPtr->cStage != 0) return (size_t)-LZ4F_ERROR_GENERIC; 403 memset(&prefNull, 0, sizeof(prefNull)); 404 if (preferencesPtr == NULL) preferencesPtr = &prefNull; 405 cctxPtr->prefs = *preferencesPtr; 406 407 /* ctx Management */ 408 { 409 U32 tableID = (cctxPtr->prefs.compressionLevel < minHClevel) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ 410 if (cctxPtr->lz4CtxLevel < tableID) 411 { 412 FREEMEM(cctxPtr->lz4CtxPtr); 413 if (cctxPtr->prefs.compressionLevel < minHClevel) 414 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); 415 else 416 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); 417 cctxPtr->lz4CtxLevel = tableID; 418 } 419 } 420 421 /* Buffer Management */ 422 if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; 423 cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); 424 425 requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); 426 if (preferencesPtr->autoFlush) 427 requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB; /* just needs dict */ 428 429 if (cctxPtr->maxBufferSize < requiredBuffSize) 430 { 431 cctxPtr->maxBufferSize = requiredBuffSize; 432 FREEMEM(cctxPtr->tmpBuff); 433 cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); 434 if (cctxPtr->tmpBuff == NULL) return (size_t)-LZ4F_ERROR_allocation_failed; 435 } 436 cctxPtr->tmpIn = cctxPtr->tmpBuff; 437 cctxPtr->tmpInSize = 0; 438 XXH32_reset(&(cctxPtr->xxh), 0); 439 if (cctxPtr->prefs.compressionLevel < minHClevel) 440 LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); 441 else 442 LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); 443 444 /* Magic Number */ 445 LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); 446 dstPtr += 4; 447 headerStart = dstPtr; 448 449 /* FLG Byte */ 450 *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ 451 + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ 452 + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ 453 + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */ 454 /* BD Byte */ 455 *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); 456 /* Optional Frame content size field */ 457 if (cctxPtr->prefs.frameInfo.contentSize) 458 { 459 LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); 460 dstPtr += 8; 461 cctxPtr->totalInSize = 0; 462 } 463 /* CRC Byte */ 464 *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); 465 dstPtr++; 466 467 cctxPtr->cStage = 1; /* header written, now request input data block */ 468 469 return (dstPtr - dstStart); 470} 471 472 473/* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. 474* The LZ4F_frameInfo_t structure is optional : 475* you can provide NULL as argument, preferences will then be set to cover worst case situations. 476* */ 477size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) 478{ 479 LZ4F_preferences_t prefsNull; 480 memset(&prefsNull, 0, sizeof(prefsNull)); 481 prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ 482 { 483 const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; 484 LZ4F_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; 485 size_t blockSize = LZ4F_getBlockSize(bid); 486 unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; 487 size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; 488 size_t blockInfo = 4; /* default, without block CRC option */ 489 size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); 490 491 return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; 492 } 493} 494 495 496typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); 497 498static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) 499{ 500 /* compress one block */ 501 BYTE* cSizePtr = (BYTE*)dst; 502 U32 cSize; 503 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); 504 LZ4F_writeLE32(cSizePtr, cSize); 505 if (cSize == 0) /* compression failed */ 506 { 507 cSize = (U32)srcSize; 508 LZ4F_writeLE32(cSizePtr, cSize + LZ4F_BLOCKUNCOMPRESSED_FLAG); 509 memcpy(cSizePtr+4, src, srcSize); 510 } 511 return cSize + 4; 512} 513 514 515static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) 516{ 517 (void) level; 518 return LZ4_compress_limitedOutput_withState(ctx, src, dst, srcSize, dstSize); 519} 520 521static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) 522{ 523 (void) level; 524 return LZ4_compress_limitedOutput_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstSize); 525} 526 527static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) 528{ 529 (void) level; 530 return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); 531} 532 533static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) 534{ 535 if (level < minHClevel) 536 { 537 if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; 538 return LZ4F_localLZ4_compress_limitedOutput_continue; 539 } 540 if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC; 541 return LZ4F_localLZ4_compressHC_limitedOutput_continue; 542} 543 544static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) 545{ 546 if (cctxPtr->prefs.compressionLevel < minHClevel) 547 return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); 548 return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); 549} 550 551typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; 552 553/* LZ4F_compressUpdate() 554* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. 555* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. 556* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) 557* You can get the minimum value of dstMaxSize by using LZ4F_compressBound() 558* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. 559* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. 560* The function outputs an error code if it fails (can be tested using LZ4F_isError()) 561*/ 562size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) 563{ 564 LZ4F_compressOptions_t cOptionsNull; 565 LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; 566 size_t blockSize = cctxPtr->maxBlockSize; 567 const BYTE* srcPtr = (const BYTE*)srcBuffer; 568 const BYTE* const srcEnd = srcPtr + srcSize; 569 BYTE* const dstStart = (BYTE*)dstBuffer; 570 BYTE* dstPtr = dstStart; 571 LZ4F_lastBlockStatus lastBlockCompressed = notDone; 572 compressFunc_t compress; 573 574 575 if (cctxPtr->cStage != 1) return (size_t)-LZ4F_ERROR_GENERIC; 576 if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; 577 memset(&cOptionsNull, 0, sizeof(cOptionsNull)); 578 if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; 579 580 /* select compression function */ 581 compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); 582 583 /* complete tmp buffer */ 584 if (cctxPtr->tmpInSize > 0) /* some data already within tmp buffer */ 585 { 586 size_t sizeToCopy = blockSize - cctxPtr->tmpInSize; 587 if (sizeToCopy > srcSize) 588 { 589 /* add src to tmpIn buffer */ 590 memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); 591 srcPtr = srcEnd; 592 cctxPtr->tmpInSize += srcSize; 593 /* still needs some CRC */ 594 } 595 else 596 { 597 /* complete tmpIn block and then compress it */ 598 lastBlockCompressed = fromTmpBuffer; 599 memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); 600 srcPtr += sizeToCopy; 601 602 dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); 603 604 if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; 605 cctxPtr->tmpInSize = 0; 606 } 607 } 608 609 while ((size_t)(srcEnd - srcPtr) >= blockSize) 610 { 611 /* compress full block */ 612 lastBlockCompressed = fromSrcBuffer; 613 dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); 614 srcPtr += blockSize; 615 } 616 617 if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) 618 { 619 /* compress remaining input < blockSize */ 620 lastBlockCompressed = fromSrcBuffer; 621 dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); 622 srcPtr = srcEnd; 623 } 624 625 /* preserve dictionary if necessary */ 626 if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) 627 { 628 if (compressOptionsPtr->stableSrc) 629 { 630 cctxPtr->tmpIn = cctxPtr->tmpBuff; 631 } 632 else 633 { 634 int realDictSize = LZ4F_localSaveDict(cctxPtr); 635 if (realDictSize==0) return (size_t)-LZ4F_ERROR_GENERIC; 636 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; 637 } 638 } 639 640 /* keep tmpIn within limits */ 641 if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */ 642 && !(cctxPtr->prefs.autoFlush)) 643 { 644 int realDictSize = LZ4F_localSaveDict(cctxPtr); 645 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; 646 } 647 648 /* some input data left, necessarily < blockSize */ 649 if (srcPtr < srcEnd) 650 { 651 /* fill tmp buffer */ 652 size_t sizeToCopy = srcEnd - srcPtr; 653 memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); 654 cctxPtr->tmpInSize = sizeToCopy; 655 } 656 657 if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) 658 XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); 659 660 cctxPtr->totalInSize += srcSize; 661 return dstPtr - dstStart; 662} 663 664 665/* LZ4F_flush() 666* Should you need to create compressed data immediately, without waiting for a block to be filled, 667* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. 668* The result of the function is the number of bytes written into dstBuffer 669* (it can be zero, this means there was no data left within compressionContext) 670* The function outputs an error code if it fails (can be tested using LZ4F_isError()) 671* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. 672*/ 673size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) 674{ 675 LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; 676 BYTE* const dstStart = (BYTE*)dstBuffer; 677 BYTE* dstPtr = dstStart; 678 compressFunc_t compress; 679 680 681 if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ 682 if (cctxPtr->cStage != 1) return (size_t)-LZ4F_ERROR_GENERIC; 683 if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ 684 (void)compressOptionsPtr; /* not yet useful */ 685 686 /* select compression function */ 687 compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); 688 689 /* compress tmp buffer */ 690 dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); 691 if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; 692 cctxPtr->tmpInSize = 0; 693 694 /* keep tmpIn within limits */ 695 if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily LZ4F_blockLinked */ 696 { 697 int realDictSize = LZ4F_localSaveDict(cctxPtr); 698 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; 699 } 700 701 return dstPtr - dstStart; 702} 703 704 705/* LZ4F_compressEnd() 706* When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). 707* It will flush whatever data remained within compressionContext (like LZ4_flush()) 708* but also properly finalize the frame, with an endMark and a checksum. 709* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) 710* The function outputs an error code if it fails (can be tested using LZ4F_isError()) 711* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. 712* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. 713*/ 714size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) 715{ 716 LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; 717 BYTE* const dstStart = (BYTE*)dstBuffer; 718 BYTE* dstPtr = dstStart; 719 size_t errorCode; 720 721 errorCode = LZ4F_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); 722 if (LZ4F_isError(errorCode)) return errorCode; 723 dstPtr += errorCode; 724 725 LZ4F_writeLE32(dstPtr, 0); 726 dstPtr+=4; /* endMark */ 727 728 if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) 729 { 730 U32 xxh = XXH32_digest(&(cctxPtr->xxh)); 731 LZ4F_writeLE32(dstPtr, xxh); 732 dstPtr+=4; /* content Checksum */ 733 } 734 735 cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ 736 737 if (cctxPtr->prefs.frameInfo.contentSize) 738 { 739 if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) 740 return (size_t)-LZ4F_ERROR_frameSize_wrong; 741 } 742 743 return dstPtr - dstStart; 744} 745 746 747/********************************** 748* Decompression functions 749**********************************/ 750 751/* Resource management */ 752 753/* LZ4F_createDecompressionContext() : 754* The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. 755* This is achieved using LZ4F_createDecompressionContext(). 756* The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. 757* If the result LZ4F_errorCode_t is not zero, there was an error during context creation. 758* Object can release its memory using LZ4F_freeDecompressionContext(); 759*/ 760LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber) 761{ 762 LZ4F_dctx_t* dctxPtr; 763 764 dctxPtr = (LZ4F_dctx_t*)ALLOCATOR(sizeof(LZ4F_dctx_t)); 765 if (dctxPtr==NULL) return (LZ4F_errorCode_t)-LZ4F_ERROR_GENERIC; 766 767 dctxPtr->version = versionNumber; 768 *LZ4F_decompressionContextPtr = (LZ4F_decompressionContext_t)dctxPtr; 769 return LZ4F_OK_NoError; 770} 771 772LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_decompressionContext) 773{ 774 LZ4F_errorCode_t result = LZ4F_OK_NoError; 775 LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)LZ4F_decompressionContext; 776 if (dctxPtr != NULL) /* can accept NULL input, like free() */ 777 { 778 result = (LZ4F_errorCode_t)dctxPtr->dStage; 779 FREEMEM(dctxPtr->tmpIn); 780 FREEMEM(dctxPtr->tmpOutBuffer); 781 FREEMEM(dctxPtr); 782 } 783 return result; 784} 785 786 787/* ******************************************************************** */ 788/* ********************* Decompression ******************************** */ 789/* ******************************************************************** */ 790 791typedef enum { dstage_getHeader=0, dstage_storeHeader, 792 dstage_getCBlockSize, dstage_storeCBlockSize, 793 dstage_copyDirect, 794 dstage_getCBlock, dstage_storeCBlock, 795 dstage_decodeCBlock, dstage_decodeCBlock_intoDst, 796 dstage_decodeCBlock_intoTmp, dstage_flushOut, 797 dstage_getSuffix, dstage_storeSuffix, 798 dstage_getSFrameSize, dstage_storeSFrameSize, 799 dstage_skipSkippable 800} dStage_t; 801 802 803/* LZ4F_decodeHeader 804 return : nb Bytes read from srcVoidPtr (necessarily <= srcSize) 805 or an error code (testable with LZ4F_isError()) 806 output : set internal values of dctx, such as 807 dctxPtr->frameInfo and dctxPtr->dStage. 808 input : srcVoidPtr points at the **beginning of the frame** 809*/ 810static size_t LZ4F_decodeHeader(LZ4F_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) 811{ 812 BYTE FLG, BD, HC; 813 unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; 814 size_t bufferNeeded; 815 size_t frameHeaderSize; 816 const BYTE* srcPtr = (const BYTE*)srcVoidPtr; 817 818 /* need to decode header to get frameInfo */ 819 if (srcSize < minFHSize) return (size_t)-LZ4F_ERROR_frameHeader_incomplete; /* minimal frame header size */ 820 memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); 821 822 /* special case : skippable frames */ 823 if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) 824 { 825 dctxPtr->frameInfo.frameType = LZ4F_skippableFrame; 826 if (srcVoidPtr == (void*)(dctxPtr->header)) 827 { 828 dctxPtr->tmpInSize = srcSize; 829 dctxPtr->tmpInTarget = 8; 830 dctxPtr->dStage = dstage_storeSFrameSize; 831 return srcSize; 832 } 833 else 834 { 835 dctxPtr->dStage = dstage_getSFrameSize; 836 return 4; 837 } 838 } 839 840 /* control magic number */ 841 if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-LZ4F_ERROR_frameType_unknown; 842 dctxPtr->frameInfo.frameType = LZ4F_frame; 843 844 /* Flags */ 845 FLG = srcPtr[4]; 846 version = (FLG>>6) & _2BITS; 847 blockMode = (FLG>>5) & _1BIT; 848 blockChecksumFlag = (FLG>>4) & _1BIT; 849 contentSizeFlag = (FLG>>3) & _1BIT; 850 contentChecksumFlag = (FLG>>2) & _1BIT; 851 852 /* Frame Header Size */ 853 frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize; 854 855 if (srcSize < frameHeaderSize) 856 { 857 /* not enough input to fully decode frame header */ 858 if (srcPtr != dctxPtr->header) 859 memcpy(dctxPtr->header, srcPtr, srcSize); 860 dctxPtr->tmpInSize = srcSize; 861 dctxPtr->tmpInTarget = frameHeaderSize; 862 dctxPtr->dStage = dstage_storeHeader; 863 return srcSize; 864 } 865 866 BD = srcPtr[5]; 867 blockSizeID = (BD>>4) & _3BITS; 868 869 /* validate */ 870 if (version != 1) return (size_t)-LZ4F_ERROR_headerVersion_wrong; /* Version Number, only supported value */ 871 if (blockChecksumFlag != 0) return (size_t)-LZ4F_ERROR_blockChecksum_unsupported; /* Not supported for the time being */ 872 if (((FLG>>0)&_2BITS) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bits */ 873 if (((BD>>7)&_1BIT) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bit */ 874 if (blockSizeID < 4) return (size_t)-LZ4F_ERROR_maxBlockSize_invalid; /* 4-7 only supported values for the time being */ 875 if (((BD>>0)&_4BITS) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bits */ 876 877 /* check */ 878 HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); 879 if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-LZ4F_ERROR_headerChecksum_invalid; /* Bad header checksum error */ 880 881 /* save */ 882 dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; 883 dctxPtr->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; 884 dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; 885 dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); 886 if (contentSizeFlag) 887 dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); 888 889 /* init */ 890 if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); 891 892 /* alloc */ 893 bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); 894 if (bufferNeeded > dctxPtr->maxBufferSize) /* tmp buffers too small */ 895 { 896 FREEMEM(dctxPtr->tmpIn); 897 FREEMEM(dctxPtr->tmpOutBuffer); 898 dctxPtr->maxBufferSize = bufferNeeded; 899 dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize); 900 if (dctxPtr->tmpIn == NULL) return (size_t)-LZ4F_ERROR_GENERIC; 901 dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize); 902 if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-LZ4F_ERROR_GENERIC; 903 } 904 dctxPtr->tmpInSize = 0; 905 dctxPtr->tmpInTarget = 0; 906 dctxPtr->dict = dctxPtr->tmpOutBuffer; 907 dctxPtr->dictSize = 0; 908 dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; 909 dctxPtr->tmpOutStart = 0; 910 dctxPtr->tmpOutSize = 0; 911 912 dctxPtr->dStage = dstage_getCBlockSize; 913 914 return frameHeaderSize; 915} 916 917 918/* LZ4F_getFrameInfo() 919* This function decodes frame header information, such as blockSize. 920* It is optional : you could start by calling directly LZ4F_decompress() instead. 921* The objective is to extract header information without starting decompression, typically for allocation purposes. 922* LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. 923* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). 924* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) 925* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress, 926* or an error code which can be tested using LZ4F_isError(). 927*/ 928LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dCtx, LZ4F_frameInfo_t* frameInfoPtr, 929 const void* srcBuffer, size_t* srcSizePtr) 930{ 931 LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)dCtx; 932 933 if (dctxPtr->dStage > dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */ 934 { 935 size_t o=0, i=0; 936 /* frameInfo already decoded */ 937 *srcSizePtr = 0; 938 *frameInfoPtr = dctxPtr->frameInfo; 939 return LZ4F_decompress(dCtx, NULL, &o, NULL, &i, NULL); 940 } 941 else 942 { 943 size_t o=0; 944 size_t nextSrcSize = LZ4F_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL); 945 if (dctxPtr->dStage <= dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */ 946 return (size_t)-LZ4F_ERROR_frameHeader_incomplete; 947 *frameInfoPtr = dctxPtr->frameInfo; 948 return nextSrcSize; 949 } 950} 951 952 953/* trivial redirector, for common prototype */ 954static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) 955{ 956 (void)dictStart; (void)dictSize; 957 return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize); 958} 959 960 961static void LZ4F_updateDict(LZ4F_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) 962{ 963 if (dctxPtr->dictSize==0) 964 dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ 965 966 if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) /* dictionary continuity */ 967 { 968 dctxPtr->dictSize += dstSize; 969 return; 970 } 971 972 if (dstPtr - dstPtr0 + dstSize >= 64 KB) /* dstBuffer large enough to become dictionary */ 973 { 974 dctxPtr->dict = (const BYTE*)dstPtr0; 975 dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; 976 return; 977 } 978 979 if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) 980 { 981 /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */ 982 dctxPtr->dictSize += dstSize; 983 return; 984 } 985 986 if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ 987 { 988 size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; 989 size_t copySize = 64 KB - dctxPtr->tmpOutSize; 990 const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; 991 if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; 992 if (copySize > preserveSize) copySize = preserveSize; 993 994 memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); 995 996 dctxPtr->dict = dctxPtr->tmpOutBuffer; 997 dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; 998 return; 999 } 1000 1001 if (dctxPtr->dict == dctxPtr->tmpOutBuffer) /* copy dst into tmp to complete dict */ 1002 { 1003 if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) /* tmp buffer not large enough */ 1004 { 1005 size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ 1006 memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); 1007 dctxPtr->dictSize = preserveSize; 1008 } 1009 memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize); 1010 dctxPtr->dictSize += dstSize; 1011 return; 1012 } 1013 1014 /* join dict & dest into tmp */ 1015 { 1016 size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ 1017 if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; 1018 memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); 1019 memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); 1020 dctxPtr->dict = dctxPtr->tmpOutBuffer; 1021 dctxPtr->dictSize = preserveSize + dstSize; 1022 } 1023} 1024 1025 1026 1027/* LZ4F_decompress() 1028* Call this function repetitively to regenerate data compressed within srcBuffer. 1029* The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. 1030* 1031* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). 1032* 1033* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). 1034* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. 1035* You will have to call it again, continuing from where it stopped. 1036* 1037* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. 1038* Basically, it's the size of the current (or remaining) compressed block + header of next block. 1039* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. 1040* Note that this is just a hint, you can always provide any srcSize you want. 1041* When a frame is fully decoded, the function result will be 0. 1042* If decompression failed, function result is an error code which can be tested using LZ4F_isError(). 1043*/ 1044size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, 1045 void* dstBuffer, size_t* dstSizePtr, 1046 const void* srcBuffer, size_t* srcSizePtr, 1047 const LZ4F_decompressOptions_t* decompressOptionsPtr) 1048{ 1049 LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)decompressionContext; 1050 LZ4F_decompressOptions_t optionsNull; 1051 const BYTE* const srcStart = (const BYTE*)srcBuffer; 1052 const BYTE* const srcEnd = srcStart + *srcSizePtr; 1053 const BYTE* srcPtr = srcStart; 1054 BYTE* const dstStart = (BYTE*)dstBuffer; 1055 BYTE* const dstEnd = dstStart + *dstSizePtr; 1056 BYTE* dstPtr = dstStart; 1057 const BYTE* selectedIn = NULL; 1058 unsigned doAnotherStage = 1; 1059 size_t nextSrcSizeHint = 1; 1060 1061 1062 memset(&optionsNull, 0, sizeof(optionsNull)); 1063 if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; 1064 *srcSizePtr = 0; 1065 *dstSizePtr = 0; 1066 1067 /* expect to continue decoding src buffer where it left previously */ 1068 if (dctxPtr->srcExpect != NULL) 1069 { 1070 if (srcStart != dctxPtr->srcExpect) return (size_t)-LZ4F_ERROR_srcPtr_wrong; 1071 } 1072 1073 /* programmed as a state machine */ 1074 1075 while (doAnotherStage) 1076 { 1077 1078 switch(dctxPtr->dStage) 1079 { 1080 1081 case dstage_getHeader: 1082 { 1083 if ((size_t)(srcEnd-srcPtr) >= maxFHSize) /* enough to decode - shortcut */ 1084 { 1085 LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); 1086 if (LZ4F_isError(errorCode)) return errorCode; 1087 srcPtr += errorCode; 1088 break; 1089 } 1090 dctxPtr->tmpInSize = 0; 1091 dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */ 1092 dctxPtr->dStage = dstage_storeHeader; 1093 } 1094 1095 case dstage_storeHeader: 1096 { 1097 size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; 1098 if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; 1099 memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); 1100 dctxPtr->tmpInSize += sizeToCopy; 1101 srcPtr += sizeToCopy; 1102 if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) 1103 { 1104 nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ 1105 doAnotherStage = 0; /* not enough src data, ask for some more */ 1106 break; 1107 } 1108 { 1109 LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); 1110 if (LZ4F_isError(errorCode)) return errorCode; 1111 } 1112 break; 1113 } 1114 1115 case dstage_getCBlockSize: 1116 { 1117 if ((size_t)(srcEnd - srcPtr) >= BHSize) 1118 { 1119 selectedIn = srcPtr; 1120 srcPtr += BHSize; 1121 } 1122 else 1123 { 1124 /* not enough input to read cBlockSize field */ 1125 dctxPtr->tmpInSize = 0; 1126 dctxPtr->dStage = dstage_storeCBlockSize; 1127 } 1128 } 1129 1130 if (dctxPtr->dStage == dstage_storeCBlockSize) 1131 case dstage_storeCBlockSize: 1132 { 1133 size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; 1134 if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; 1135 memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); 1136 srcPtr += sizeToCopy; 1137 dctxPtr->tmpInSize += sizeToCopy; 1138 if (dctxPtr->tmpInSize < BHSize) /* not enough input to get full cBlockSize; wait for more */ 1139 { 1140 nextSrcSizeHint = BHSize - dctxPtr->tmpInSize; 1141 doAnotherStage = 0; 1142 break; 1143 } 1144 selectedIn = dctxPtr->tmpIn; 1145 } 1146 1147 /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ 1148 { 1149 size_t nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; 1150 if (nextCBlockSize==0) /* frameEnd signal, no more CBlock */ 1151 { 1152 dctxPtr->dStage = dstage_getSuffix; 1153 break; 1154 } 1155 if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-LZ4F_ERROR_GENERIC; /* invalid cBlockSize */ 1156 dctxPtr->tmpInTarget = nextCBlockSize; 1157 if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) 1158 { 1159 dctxPtr->dStage = dstage_copyDirect; 1160 break; 1161 } 1162 dctxPtr->dStage = dstage_getCBlock; 1163 if (dstPtr==dstEnd) 1164 { 1165 nextSrcSizeHint = nextCBlockSize + BHSize; 1166 doAnotherStage = 0; 1167 } 1168 break; 1169 } 1170 1171 case dstage_copyDirect: /* uncompressed block */ 1172 { 1173 size_t sizeToCopy = dctxPtr->tmpInTarget; 1174 if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ 1175 if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; 1176 memcpy(dstPtr, srcPtr, sizeToCopy); 1177 if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); 1178 if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy; 1179 1180 /* dictionary management */ 1181 if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) 1182 LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); 1183 1184 srcPtr += sizeToCopy; 1185 dstPtr += sizeToCopy; 1186 if (sizeToCopy == dctxPtr->tmpInTarget) /* all copied */ 1187 { 1188 dctxPtr->dStage = dstage_getCBlockSize; 1189 break; 1190 } 1191 dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ 1192 nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize; 1193 doAnotherStage = 0; 1194 break; 1195 } 1196 1197 case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ 1198 { 1199 if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) 1200 { 1201 dctxPtr->tmpInSize = 0; 1202 dctxPtr->dStage = dstage_storeCBlock; 1203 break; 1204 } 1205 selectedIn = srcPtr; 1206 srcPtr += dctxPtr->tmpInTarget; 1207 dctxPtr->dStage = dstage_decodeCBlock; 1208 break; 1209 } 1210 1211 case dstage_storeCBlock: 1212 { 1213 size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; 1214 if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; 1215 memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); 1216 dctxPtr->tmpInSize += sizeToCopy; 1217 srcPtr += sizeToCopy; 1218 if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* need more input */ 1219 { 1220 nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; 1221 doAnotherStage=0; 1222 break; 1223 } 1224 selectedIn = dctxPtr->tmpIn; 1225 dctxPtr->dStage = dstage_decodeCBlock; 1226 break; 1227 } 1228 1229 case dstage_decodeCBlock: 1230 { 1231 if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ 1232 dctxPtr->dStage = dstage_decodeCBlock_intoTmp; 1233 else 1234 dctxPtr->dStage = dstage_decodeCBlock_intoDst; 1235 break; 1236 } 1237 1238 case dstage_decodeCBlock_intoDst: 1239 { 1240 int (*decoder)(const char*, char*, int, int, const char*, int); 1241 int decodedSize; 1242 1243 if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) 1244 decoder = LZ4_decompress_safe_usingDict; 1245 else 1246 decoder = LZ4F_decompress_safe; 1247 1248 decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); 1249 if (decodedSize < 0) return (size_t)-LZ4F_ERROR_GENERIC; /* decompression failed */ 1250 if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); 1251 if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; 1252 1253 /* dictionary management */ 1254 if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) 1255 LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); 1256 1257 dstPtr += decodedSize; 1258 dctxPtr->dStage = dstage_getCBlockSize; 1259 break; 1260 } 1261 1262 case dstage_decodeCBlock_intoTmp: 1263 { 1264 /* not enough place into dst : decode into tmpOut */ 1265 int (*decoder)(const char*, char*, int, int, const char*, int); 1266 int decodedSize; 1267 1268 if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) 1269 decoder = LZ4_decompress_safe_usingDict; 1270 else 1271 decoder = LZ4F_decompress_safe; 1272 1273 /* ensure enough place for tmpOut */ 1274 if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) 1275 { 1276 if (dctxPtr->dict == dctxPtr->tmpOutBuffer) 1277 { 1278 if (dctxPtr->dictSize > 128 KB) 1279 { 1280 memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); 1281 dctxPtr->dictSize = 64 KB; 1282 } 1283 dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize; 1284 } 1285 else /* dict not within tmp */ 1286 { 1287 size_t reservedDictSpace = dctxPtr->dictSize; 1288 if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB; 1289 dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; 1290 } 1291 } 1292 1293 /* Decode */ 1294 decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); 1295 if (decodedSize < 0) return (size_t)-LZ4F_ERROR_decompressionFailed; /* decompression failed */ 1296 if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); 1297 if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; 1298 dctxPtr->tmpOutSize = decodedSize; 1299 dctxPtr->tmpOutStart = 0; 1300 dctxPtr->dStage = dstage_flushOut; 1301 break; 1302 } 1303 1304 case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ 1305 { 1306 size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; 1307 if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; 1308 memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); 1309 1310 /* dictionary management */ 1311 if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) 1312 LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); 1313 1314 dctxPtr->tmpOutStart += sizeToCopy; 1315 dstPtr += sizeToCopy; 1316 1317 /* end of flush ? */ 1318 if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) 1319 { 1320 dctxPtr->dStage = dstage_getCBlockSize; 1321 break; 1322 } 1323 nextSrcSizeHint = BHSize; 1324 doAnotherStage = 0; /* still some data to flush */ 1325 break; 1326 } 1327 1328 case dstage_getSuffix: 1329 { 1330 size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; 1331 if (dctxPtr->frameRemainingSize) return (size_t)-LZ4F_ERROR_frameSize_wrong; /* incorrect frame size decoded */ 1332 if (suffixSize == 0) /* frame completed */ 1333 { 1334 nextSrcSizeHint = 0; 1335 dctxPtr->dStage = dstage_getHeader; 1336 doAnotherStage = 0; 1337 break; 1338 } 1339 if ((srcEnd - srcPtr) < 4) /* not enough size for entire CRC */ 1340 { 1341 dctxPtr->tmpInSize = 0; 1342 dctxPtr->dStage = dstage_storeSuffix; 1343 } 1344 else 1345 { 1346 selectedIn = srcPtr; 1347 srcPtr += 4; 1348 } 1349 } 1350 1351 if (dctxPtr->dStage == dstage_storeSuffix) 1352 case dstage_storeSuffix: 1353 { 1354 size_t sizeToCopy = 4 - dctxPtr->tmpInSize; 1355 if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; 1356 memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); 1357 srcPtr += sizeToCopy; 1358 dctxPtr->tmpInSize += sizeToCopy; 1359 if (dctxPtr->tmpInSize < 4) /* not enough input to read complete suffix */ 1360 { 1361 nextSrcSizeHint = 4 - dctxPtr->tmpInSize; 1362 doAnotherStage=0; 1363 break; 1364 } 1365 selectedIn = dctxPtr->tmpIn; 1366 } 1367 1368 /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ 1369 { 1370 U32 readCRC = LZ4F_readLE32(selectedIn); 1371 U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); 1372 if (readCRC != resultCRC) return (size_t)-LZ4F_ERROR_contentChecksum_invalid; 1373 nextSrcSizeHint = 0; 1374 dctxPtr->dStage = dstage_getHeader; 1375 doAnotherStage = 0; 1376 break; 1377 } 1378 1379 case dstage_getSFrameSize: 1380 { 1381 if ((srcEnd - srcPtr) >= 4) 1382 { 1383 selectedIn = srcPtr; 1384 srcPtr += 4; 1385 } 1386 else 1387 { 1388 /* not enough input to read cBlockSize field */ 1389 dctxPtr->tmpInSize = 4; 1390 dctxPtr->tmpInTarget = 8; 1391 dctxPtr->dStage = dstage_storeSFrameSize; 1392 } 1393 } 1394 1395 if (dctxPtr->dStage == dstage_storeSFrameSize) 1396 case dstage_storeSFrameSize: 1397 { 1398 size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; 1399 if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; 1400 memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); 1401 srcPtr += sizeToCopy; 1402 dctxPtr->tmpInSize += sizeToCopy; 1403 if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* not enough input to get full sBlockSize; wait for more */ 1404 { 1405 nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; 1406 doAnotherStage = 0; 1407 break; 1408 } 1409 selectedIn = dctxPtr->header + 4; 1410 } 1411 1412 /* case dstage_decodeSFrameSize: */ /* no direct access */ 1413 { 1414 size_t SFrameSize = LZ4F_readLE32(selectedIn); 1415 dctxPtr->frameInfo.contentSize = SFrameSize; 1416 dctxPtr->tmpInTarget = SFrameSize; 1417 dctxPtr->dStage = dstage_skipSkippable; 1418 break; 1419 } 1420 1421 case dstage_skipSkippable: 1422 { 1423 size_t skipSize = dctxPtr->tmpInTarget; 1424 if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr; 1425 srcPtr += skipSize; 1426 dctxPtr->tmpInTarget -= skipSize; 1427 doAnotherStage = 0; 1428 nextSrcSizeHint = dctxPtr->tmpInTarget; 1429 if (nextSrcSizeHint) break; 1430 dctxPtr->dStage = dstage_getHeader; 1431 break; 1432 } 1433 } 1434 } 1435 1436 /* preserve dictionary within tmp if necessary */ 1437 if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) 1438 &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) 1439 &&(!decompressOptionsPtr->stableDst) 1440 &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) 1441 ) 1442 { 1443 if (dctxPtr->dStage == dstage_flushOut) 1444 { 1445 size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; 1446 size_t copySize = 64 KB - dctxPtr->tmpOutSize; 1447 const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; 1448 if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; 1449 if (copySize > preserveSize) copySize = preserveSize; 1450 1451 memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); 1452 1453 dctxPtr->dict = dctxPtr->tmpOutBuffer; 1454 dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart; 1455 } 1456 else 1457 { 1458 size_t newDictSize = dctxPtr->dictSize; 1459 const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; 1460 if ((newDictSize) > 64 KB) newDictSize = 64 KB; 1461 1462 memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); 1463 1464 dctxPtr->dict = dctxPtr->tmpOutBuffer; 1465 dctxPtr->dictSize = newDictSize; 1466 dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; 1467 } 1468 } 1469 1470 /* require function to be called again from position where it stopped */ 1471 if (srcPtr<srcEnd) 1472 dctxPtr->srcExpect = srcPtr; 1473 else 1474 dctxPtr->srcExpect = NULL; 1475 1476 *srcSizePtr = (srcPtr - srcStart); 1477 *dstSizePtr = (dstPtr - dstStart); 1478 return nextSrcSizeHint; 1479} 1480