diff --git a/NEWS b/NEWS index 167666c4a..ace6d4826 100644 --- a/NEWS +++ b/NEWS @@ -4,8 +4,10 @@ added : NetBSD install target (#338) Improved : variable compression speed improvements on batches of small files. Fixed : CLI -d output to stdout by default when input is stdin (#322) Fixed : CLI correctly detects console on Mac OS-X -Fixed : Legacy decoders use unified error codes (#341), reported by benrg +Fixed : CLI supports recursive mode `-r` on Mac OS-X +Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) +Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) Fixed : zstd-pgo, reported by octoploid (#329) v1.0.0 diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f832e081a..f5b712fc3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -113,6 +113,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { + if (cctx==NULL) return 0; /* support sizeof on NULL */ return sizeof(*cctx) + cctx->workSpaceSize; } @@ -168,7 +169,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1< srcLog) cPar.windowLog = srcLog; } } if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; @@ -238,9 +239,9 @@ typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_comp note : 'params' must be validated */ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ZSTD_parameters params, U64 frameContentSize, - ZSTD_compResetPolicy_e crp) + ZSTD_compResetPolicy_e const crp) { - if (crp == ZSTDcrp_continue) /* still some issues */ + if (crp == ZSTDcrp_continue) if (ZSTD_equivalentParams(params, zc->params)) return ZSTD_continueCCtx(zc, params, frameContentSize); @@ -2680,6 +2681,12 @@ struct ZSTD_CDict_s { ZSTD_CCtx* refContext; }; /* typedef'd tp ZSTD_CDict within "zstd.h" */ +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize; +} + ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) { if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; @@ -2731,17 +2738,23 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) } } +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) +{ + if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext)) + else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize)); + return 0; +} + /*! ZSTD_compress_usingCDict() : * Compression using a digested Dictionary. * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. * Note that compression level is decided during dictionary creation */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict) +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) { - if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext)) - else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, srcSize)); + CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); if (cdict->refContext->params.fParams.contentSizeFlag==1) { cctx->params.fParams.contentSizeFlag = 1; @@ -2760,7 +2773,8 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; struct ZSTD_CStream_s { - ZSTD_CCtx* zc; + ZSTD_CCtx* cctx; + ZSTD_CDict* cdict; char* inBuff; size_t inBuffSize; size_t inToCompress; @@ -2793,8 +2807,8 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) if (zcs==NULL) return NULL; memset(zcs, 0, sizeof(ZSTD_CStream)); memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); - zcs->zc = ZSTD_createCCtx_advanced(customMem); - if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; } + zcs->cctx = ZSTD_createCCtx_advanced(customMem); + if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; } return zcs; } @@ -2802,7 +2816,8 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) { if (zcs==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = zcs->customMem; - ZSTD_freeCCtx(zcs->zc); + ZSTD_freeCCtx(zcs->cctx); + ZSTD_freeCDict(zcs->cdict); ZSTD_free(zcs->inBuff, cMem); ZSTD_free(zcs->outBuff, cMem); ZSTD_free(zcs, cMem); @@ -2816,6 +2831,19 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +{ + CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)); + + zcs->inToCompress = 0; + zcs->inBuffPos = 0; + zcs->inBuffTarget = zcs->blockSize; + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + zcs->frameEnded = 0; + return 0; /* ready to go */ +} + size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) @@ -2837,16 +2865,13 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, if (zcs->outBuff == NULL) return ERROR(memory_allocation); } - CHECK_F(ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize)); + ZSTD_freeCDict(zcs->cdict); + zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); + if (zcs->cdict == NULL) return ERROR(memory_allocation); - zcs->inToCompress = 0; - zcs->inBuffPos = 0; - zcs->inBuffTarget = zcs->blockSize; - zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_load; zcs->checksum = params.fParams.checksumFlag > 0; - zcs->frameEnded = 0; - return 0; /* ready to go */ + + return ZSTD_resetCStream(zcs, pledgedSrcSize); } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) @@ -2862,7 +2887,8 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) { - return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize; + if (zcs==NULL) return 0; /* support sizeof on NULL */ + return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize; } /*====== Compression ======*/ @@ -2913,8 +2939,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, else cDst = zcs->outBuff, oSize = zcs->outBuffSize; cSize = (flush == zsf_end) ? - ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : - ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); + ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); if (ZSTD_isError(cSize)) return cSize; if (flush == zsf_end) zcs->frameEnded = 1; /* prepare next block */ @@ -3008,7 +3034,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) /* create epilogue */ zcs->stage = zcss_final; zcs->outBuffContentSize = !notEnded ? 0 : - ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ + ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ } /* flush epilogue */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index c6bb5329c..2b2539a4a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -80,8 +80,12 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, struct ZSTD_DCtx_s { + const FSE_DTable* LLTptr; + const FSE_DTable* MLTptr; + const FSE_DTable* OFTptr; + const HUF_DTable* HUFptr; FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; - FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; + FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ const void* previousDstEnd; @@ -107,7 +111,7 @@ struct ZSTD_DCtx_s BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ -size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; return sizeof(ZSTD_DCtx); } /* support sizeof on NULL */ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } @@ -119,11 +123,15 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->base = NULL; dctx->vBase = NULL; dctx->dictEnd = NULL; - dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); + dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue)); - memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); + memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->LLTable; + dctx->MLTptr = dctx->MLTable; + dctx->OFTptr = dctx->OFTable; + dctx->HUFptr = dctx->hufTable; return 0; } @@ -159,6 +167,25 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ } +static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + ZSTD_decompressBegin(dstDCtx); /* init */ + dstDCtx->dictEnd = srcDCtx->dictEnd; + dstDCtx->vBase = srcDCtx->vBase; + dstDCtx->base = srcDCtx->base; + dstDCtx->previousDstEnd = srcDCtx->previousDstEnd; + dstDCtx->dictID = srcDCtx->dictID; + dstDCtx->litEntropy = srcDCtx->litEntropy; + dstDCtx->fseEntropy = srcDCtx->fseEntropy; + dstDCtx->LLTptr = srcDCtx->LLTable; + dstDCtx->MLTptr = srcDCtx->MLTable; + dstDCtx->OFTptr = srcDCtx->OFTable; + dstDCtx->HUFptr = srcDCtx->hufTable; + dstDCtx->rep[0] = srcDCtx->rep[0]; + dstDCtx->rep[1] = srcDCtx->rep[1]; + dstDCtx->rep[2] = srcDCtx->rep[2]; +} + /*-************************************************************* * Decompression section @@ -350,34 +377,31 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, { case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ /* 2 - 2 - 10 - 10 */ - { singleStream = !lhlCode; - lhSize = 3; - litSize = (lhc >> 4) & 0x3FF; - litCSize = (lhc >> 14) & 0x3FF; - break; - } + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; case 2: /* 2 - 2 - 14 - 14 */ - { lhSize = 4; - litSize = (lhc >> 4) & 0x3FFF; - litCSize = lhc >> 18; - break; - } + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; case 3: /* 2 - 2 - 18 - 18 */ - { lhSize = 5; - litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + (istart[4] << 10); - break; - } + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + (istart[4] << 10); + break; } if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); if (HUF_isError((litEncType==set_repeat) ? ( singleStream ? - HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) : - HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) : + HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) : + HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) : ( singleStream ? HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) @@ -387,6 +411,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable; return litCSize + lhSize; } @@ -461,7 +486,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, @return : nb bytes read from src, or an error code if it fails, testable with ZSTD_isError() */ -FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog, +static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog, const void* src, size_t srcSize, const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable) { @@ -491,8 +516,7 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e } -size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, - FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 flagRepeatTable, +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, const void* src, size_t srcSize) { const BYTE* const istart = (const BYTE* const)src; @@ -522,16 +546,19 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, ip++; /* Build DTables */ - { size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); + { size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + if (LLtype != set_repeat) dctx->LLTptr = dctx->LLTable; ip += llhSize; } - { size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + if (OFtype != set_repeat) dctx->OFTptr = dctx->OFTable; ip += ofhSize; } - { size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + if (MLtype != set_repeat) dctx->MLTptr = dctx->MLTable; ip += mlhSize; } } @@ -714,16 +741,13 @@ static size_t ZSTD_decompressSequences( const BYTE* litPtr = dctx->litPtr; const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; const BYTE* const litEnd = litPtr + dctx->litSize; - FSE_DTable* DTableLL = dctx->LLTable; - FSE_DTable* DTableML = dctx->MLTable; - FSE_DTable* DTableOffb = dctx->OffTable; const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); int nbSeq; /* Build Decoding Tables */ - { size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize); + { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); if (ZSTD_isError(seqHSize)) return seqHSize; ip += seqHSize; } @@ -733,10 +757,10 @@ static size_t ZSTD_decompressSequences( seqState_t seqState; dctx->fseEntropy = 1; { U32 i; for (i=0; irep[i]; } - CHECK_E(BIT_initDStream(&(seqState.DStream), ip, iend-ip), corruption_detected); - FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); - FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); - FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { nbSeq--; @@ -894,25 +918,10 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } -/*! ZSTD_decompress_usingPreparedDCtx() : -* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - ZSTD_copyDCtx(dctx, refDCtx); - ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); -} - - size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict, size_t dictSize) + const void* src, size_t srcSize, + const void* dict, size_t dictSize) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); @@ -1120,7 +1129,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); + CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); dictPtr += offcodeHeaderSize; } @@ -1172,7 +1181,6 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict return ZSTD_refDictContent(dctx, dict, dictSize); } - size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { CHECK_F(ZSTD_decompressBegin(dctx)); @@ -1181,6 +1189,8 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t } +/* ====== ZSTD_DDict ====== */ + struct ZSTD_DDict_s { void* dict; size_t dictSize; @@ -1239,20 +1249,27 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict) } } +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize; +} + + /*! ZSTD_decompress_usingDDict() : * Decompression using a pre-digested Dictionary * Use dictionary without significant overhead. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict) +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); #endif - return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, - dst, dstCapacity, - src, srcSize); + ZSTD_refDCtx(dctx, ddict->refContext); + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); } @@ -1265,7 +1282,8 @@ typedef enum { zdss_init, zdss_loadHeader, /* *** Resource management *** */ struct ZSTD_DStream_s { - ZSTD_DCtx* zd; + ZSTD_DCtx* dctx; + ZSTD_DDict* ddict; ZSTD_frameParams fParams; ZSTD_dStreamStage stage; char* inBuff; @@ -1277,12 +1295,9 @@ struct ZSTD_DStream_s { size_t outStart; size_t outEnd; size_t blockSize; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ size_t lhSize; ZSTD_customMem customMem; - void* dictContent; - size_t dictSize; - const void* dictSource; void* legacyContext; U32 previousLegacyVersion; U32 legacyVersion; @@ -1306,8 +1321,8 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) if (zds==NULL) return NULL; memset(zds, 0, sizeof(ZSTD_DStream)); memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); - zds->zd = ZSTD_createDCtx_advanced(customMem); - if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } + zds->dctx = ZSTD_createDCtx_advanced(customMem); + if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; } zds->stage = zdss_init; zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; return zds; @@ -1317,10 +1332,10 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) { if (zds==NULL) return 0; /* support free on null */ { ZSTD_customMem const cMem = zds->customMem; - ZSTD_freeDCtx(zds->zd); + ZSTD_freeDCtx(zds->dctx); + ZSTD_freeDDict(zds->ddict); ZSTD_free(zds->inBuff, cMem); ZSTD_free(zds->outBuff, cMem); - ZSTD_free(zds->dictContent, cMem); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (zds->legacyContext) ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); @@ -1340,15 +1355,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di { zds->stage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) { /* new dictionary */ - if (dictSize > zds->dictSize) { - ZSTD_free(zds->dictContent, zds->customMem); - zds->dictContent = ZSTD_malloc(dictSize, zds->customMem); - if (zds->dictContent == NULL) return ERROR(memory_allocation); - } - memcpy(zds->dictContent, dict, dictSize); - zds->dictSize = dictSize; - } + ZSTD_freeDDict(zds->ddict); + zds->ddict = ZSTD_createDDict(dict, dictSize); + if (zds->ddict == NULL) return ERROR(memory_allocation); zds->legacyVersion = 0; zds->hostageByte = 0; return ZSTD_frameHeaderSize_prefix; @@ -1359,6 +1368,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } +size_t ZSTD_resetDStream(ZSTD_DStream* zds) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + zds->legacyVersion = 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; +} + size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue) { @@ -1373,7 +1391,8 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { - return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize + zds->dictSize; + if (zds==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize; } @@ -1415,7 +1434,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); if (legacyVersion) { CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, - zds->dictContent, zds->dictSize)); + zds->ddict->dict, zds->ddict->dictSize)); zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); } else { @@ -1437,11 +1456,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } /* Consume header */ - ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize); - { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_prefix */ - CHECK_F(ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size)); - { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd); - CHECK_F(ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size)); + ZSTD_refDCtx(zds->dctx, zds->ddict->refContext); + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); + { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size)); } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); @@ -1467,15 +1486,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* pass-through */ case zdss_read: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); if (neededInSize==0) { /* end of frame */ zds->stage = zdss_init; someMoreWork = 0; break; } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ - const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); - size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -1491,7 +1510,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } case zdss_load: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ size_t loadedSize; if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ @@ -1501,8 +1520,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ - { const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); - size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + { const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, zds->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -1534,7 +1553,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* result */ input->pos += (size_t)(ip-istart); output->pos += (size_t)(op-ostart); - { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); if (!nextSrcSizeHint) { /* frame fully decoded */ if (zds->outEnd == zds->outStart) { /* output fully flushed */ if (zds->hostageByte) { @@ -1549,7 +1568,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } return 1; } - nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); /* preload header of next block */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ nextSrcSizeHint -= zds->inPos; /* already loaded*/ return nextSrcSizeHint; diff --git a/lib/zstd.h b/lib/zstd.h index 5cc40c63c..f79a5dcac 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -48,40 +48,41 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); * Simple API ***************************************/ /*! ZSTD_compress() : - Compresses `src` buffer into already allocated `dst`. + Compresses `src` content as a single zstd compressed frame into already allocated `dst`. Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - @return : the number of bytes written into `dst` (<= `dstCapacity), + @return : compressed size written into `dst` (<= `dstCapacity), or an error code if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); -/*! ZSTD_getDecompressedSize() : -* @return : decompressed size as a 64-bits value _if known_, 0 otherwise. -* note 1 : decompressed size can be very large (64-bits value), -* potentially larger than what local system can handle as a single memory segment. -* In which case, it's necessary to use streaming mode to decompress data. -* note 2 : decompressed size is an optional field, that may not be present. -* When `return==0`, data to decompress can have any size. -* In which case, it's necessary to use streaming mode to decompress data. -* Optionally, application may rely on its own implied limits. -* (For example, application data could be necessarily cut into blocks <= 16 KB). -* note 3 : decompressed size could be wrong or intentionally modified ! -* Always ensure result fits within application's authorized limits ! -* Each application can set its own limits. -* note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ -ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); - /*! ZSTD_decompress() : - `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ). - If originalSize is unknown, and if there is no implied application-specific limitations, - it's preferable to use streaming mode to decompress data. + `compressedSize` : must be the _exact_ size of a single compressed frame. + `dstCapacity` is an upper bound of originalSize. + If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), or an errorCode if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +/*! ZSTD_getDecompressedSize() : +* 'src' is the start of a zstd compressed frame. +* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. +* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. +* When `return==0`, data to decompress could be any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application can still use ZSTD_decompress() while relying on implied limits. +* (For example, data may be necessarily cut into blocks <= 16 KB). +* note 2 : decompressed size is always present when compression is done with ZSTD_compress() +* note 3 : decompressed size can be very large (64-bits value), +* potentially larger than what local system can handle as a single memory segment. +* In which case, it's necessary to use streaming mode to decompress data. +* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. +* Always ensure result fits within application's authorized limits. +* Each application can set its own limits. +* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + /*====== Helper functions ======*/ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ @@ -351,14 +352,18 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_sizeofCCtx() : + * Gives the amount of memory used by a given ZSTD_CCtx */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); + /*! ZSTD_createCDict_advanced() : * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem); -/*! ZSTD_sizeofCCtx() : - * Gives the amount of memory used by a given ZSTD_CCtx */ -ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +/*! ZSTD_sizeof_CDict() : + * Gives the amount of memory used by a given ZSTD_sizeof_CDict */ +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. @@ -398,10 +403,14 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -/*! ZSTD_sizeofDCtx() : +/*! ZSTD_sizeof_DCtx() : * Gives the amount of memory used by a given ZSTD_DCtx */ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +/*! ZSTD_sizeof_DDict() : + * Gives the amount of memory used by a given ZSTD_DDict */ +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + /* ****************************************************************** * Advanced Streaming functions @@ -412,7 +421,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); + ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ +ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); @@ -423,6 +433,7 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); +ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); diff --git a/programs/datagen.c b/programs/datagen.c index 109b8e3d8..1af5a38a5 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -20,10 +20,9 @@ /*-************************************ * Dependencies **************************************/ -#include /* malloc */ +#include /* malloc, free */ #include /* FILE, fwrite, fprintf */ #include /* memcpy */ -#include /* errno */ #include "mem.h" /* U32 */ @@ -176,7 +175,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba, BYTE ldt[LTSIZE]; /* literals distribution table */ /* init */ - if (buff==NULL) { fprintf(stderr, "datagen: error: %s \n", strerror(errno)); exit(1); } + if (buff==NULL) { perror("datagen"); exit(1); } if (litProba<=0.0) litProba = matchProba / 4.5; memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */ RDG_fillLiteralDistrib(ldt, litProba); diff --git a/programs/fileio.c b/programs/fileio.c index 056958245..7dee7c11b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -119,8 +119,6 @@ static clock_t g_time = 0; ***************************************/ static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } -static U32 g_maxWLog = 23; -void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; } static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } static U32 g_dictIDFlag = 1; @@ -167,7 +165,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName) return f; } -/* `dstFileName must` be non-NULL */ +/** FIO_openDstFile() : + * condition : `dstFileName` must be non-NULL. + * @result : FILE* to `dstFileName`, or NULL if it fails */ static FILE* FIO_openDstFile(const char* dstFileName) { FILE* f; @@ -250,14 +250,12 @@ typedef struct { size_t srcBufferSize; void* dstBuffer; size_t dstBufferSize; - void* dictBuffer; - size_t dictBufferSize; ZSTD_CStream* cctx; FILE* dstFile; FILE* srcFile; } cRess_t; -static cRess_t FIO_createCResources(const char* dictFileName) +static cRess_t FIO_createCResources(const char* dictFileName, int cLevel) { cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -271,19 +269,27 @@ static cRess_t FIO_createCResources(const char* dictFileName) if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory"); /* dictionary */ - ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName); + { void* dictBuffer; + size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName); + if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer"); + { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictBuffSize); + params.fParams.contentSizeFlag = 1; + params.fParams.checksumFlag = g_checksumFlag; + params.fParams.noDictIDFlag = !g_dictIDFlag; + { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, 0); + if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); + } } + free(dictBuffer); + } return ress; } static void FIO_freeCResources(cRess_t ress) { - size_t errorCode; free(ress.srcBuffer); free(ress.dstBuffer); - free(ress.dictBuffer); - errorCode = ZSTD_freeCStream(ress.cctx); - if (ZSTD_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZSTD_CStream : %s", ZSTD_getErrorName(errorCode)); + ZSTD_freeCStream(ress.cctx); /* never fails */ } @@ -293,8 +299,7 @@ static void FIO_freeCResources(cRess_t ress) * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_internal(cRess_t ress, - const char* dstFileName, const char* srcFileName, - int cLevel) + const char* dstFileName, const char* srcFileName) { FILE* const srcFile = ress.srcFile; FILE* const dstFile = ress.dstFile; @@ -303,17 +308,9 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 const fileSize = UTIL_getFileSize(srcFileName); /* init */ - { ZSTD_parameters params = ZSTD_getParams(cLevel, fileSize, ress.dictBufferSize); - params.fParams.contentSizeFlag = 1; - params.fParams.checksumFlag = g_checksumFlag; - params.fParams.noDictIDFlag = !g_dictIDFlag; - if ((g_maxWLog) && (params.cParams.windowLog > g_maxWLog)) { - params.cParams.windowLog = g_maxWLog; - params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize); - } - { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); - if (ZSTD_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(errorCode)); - } } + { size_t const resetError = ZSTD_resetCStream(ress.cctx, fileSize); + if (ZSTD_isError(resetError)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(resetError)); + } /* Main compression loop */ while (1) { @@ -367,8 +364,7 @@ static int FIO_compressFilename_internal(cRess_t ress, * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_srcFile(cRess_t ress, - const char* dstFileName, const char* srcFileName, - int cLevel) + const char* dstFileName, const char* srcFileName) { int result; @@ -380,10 +376,10 @@ static int FIO_compressFilename_srcFile(cRess_t ress, ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); + result = FIO_compressFilename_internal(ress, dstFileName, srcFileName); fclose(ress.srcFile); - if ((g_removeSrcFile) && (!result)) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } + if (g_removeSrcFile && !result) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ return result; } @@ -393,17 +389,16 @@ static int FIO_compressFilename_srcFile(cRess_t ress, * 1 : pb */ static int FIO_compressFilename_dstFile(cRess_t ress, - const char* dstFileName, const char* srcFileName, - int cLevel) + const char* dstFileName, const char* srcFileName) { int result; ress.dstFile = FIO_openDstFile(dstFileName); - if (ress.dstFile==0) return 1; + if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ - result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel); + result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName); - if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } + if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */ if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */ return result; } @@ -414,8 +409,8 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, { clock_t const start = clock(); - cRess_t const ress = FIO_createCResources(dictFileName); - int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); + cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel); + int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); @@ -433,7 +428,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile size_t dfnSize = FNSPACE; char* dstFileName = (char*)malloc(FNSPACE); size_t const suffixSize = suffix ? strlen(suffix) : 0; - cRess_t ress = FIO_createCResources(dictFileName); + cRess_t ress = FIO_createCResources(dictFileName, compressionLevel); /* init */ if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); @@ -445,8 +440,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile ress.dstFile = stdout; SET_BINARY_MODE(stdout); for (u=0; u= alreadyLoaded */ diff --git a/programs/fileio.h b/programs/fileio.h index 66805f79c..1e89aec27 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -8,7 +8,8 @@ */ -#pragma once +#ifndef FILEIO_H_23981798732 +#define FILEIO_H_23981798732 #if defined (__cplusplus) extern "C" { @@ -32,7 +33,6 @@ extern "C" { ***************************************/ void FIO_overwriteMode(void); void FIO_setNotificationLevel(unsigned level); -void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setDictIDFlag(unsigned dictIDFlag); void FIO_setChecksumFlag(unsigned checksumFlag); @@ -70,3 +70,5 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles #if defined (__cplusplus) } #endif + +#endif /* FILEIO_H_23981798732 */ diff --git a/programs/util.h b/programs/util.h index 571fc0a9d..9d28c82d7 100644 --- a/programs/util.h +++ b/programs/util.h @@ -274,7 +274,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ return nbFiles; } -#elif (defined(__unix__) || defined(__unix) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) /* snprintf, opendir */ +#elif (defined(__APPLE__) && defined(__MACH__)) || \ + ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ # include diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e829c93d6..4a58b05b8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -262,7 +262,7 @@ int main(int argCount, char** argv) if (!strcmp(argument, "--verbose")) { displayLevel++; continue; } if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; } - if (!strcmp(argument, "--ultra")) { ultra=1; FIO_setMaxWLog(0); continue; } + if (!strcmp(argument, "--ultra")) { ultra=1; continue; } if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; } if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } @@ -334,8 +334,10 @@ int main(int argCount, char** argv) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; argument++; break; +#ifdef UTIL_HAS_CREATEFILELIST /* recursive */ case 'r': recursive=1; argument++; break; +#endif #ifndef ZSTD_NOBENCH /* Benchmark */ diff --git a/tests/datagencli.c b/tests/datagencli.c index 772e3dc99..2f3ebc4d6 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -9,7 +9,7 @@ /*-************************************ -* Includes +* Dependencies **************************************/ #include "util.h" /* Compiler options */ #include /* fprintf, stderr */ diff --git a/tests/fullbench.c b/tests/fullbench.c index eaf1fc774..670b51681 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -121,13 +121,12 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co } extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); -extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 tableRepeatFlag, const void* src, size_t srcSize); +extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize); size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { - U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */ int nbSeq; (void)src; (void)srcSize; (void)dst; (void)dstSize; - return ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, 0, buff2, g_cSize); + return ZSTD_decodeSeqHeaders(g_zdc, &nbSeq, buff2, g_cSize); } @@ -289,6 +288,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) } iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ ip += ZSTD_blockHeaderSize; /* skip block header */ + ZSTD_decompressBegin(g_zdc); ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */ g_cSize = iend-ip; memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 323854b32..b8f102a9c 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -57,7 +57,7 @@ static U32 g_displayLevel = 2; if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stdout); } } -static const clock_t g_refreshRate = CLOCKS_PER_SEC * 150 / 1000; +static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_displayClock = 0; diff --git a/tests/playTests.sh b/tests/playTests.sh index 21e98bf90..042197c2d 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -205,7 +205,7 @@ $MD5SUM dirTestDict/* > tmph1 $ZSTD -f --rm dirTestDict/* -D tmpDictC $ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default if [[ "$OSTYPE" == "darwin"* ]]; then - $ECHO "test skipped on OS-X" # not compatible with OS-X's md5 + $ECHO "md5sum -c not supported on OS-X : test skipped" # not compatible with OS-X's md5 else $MD5SUM -c tmph1 fi diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 97fbaa18e..d10d4f125 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -209,7 +209,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Byte-by-byte decompression test */ DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); - { size_t r = 1; + { /* skippable frame */ + size_t r = 1; ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); inBuff.src = compressedBuffer; outBuff.dst = decodedBuffer; @@ -221,9 +222,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (ZSTD_isError(r)) goto _output_error; } + /* normal frame */ ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); r=1; - while (r) { /* normal frame */ + while (r) { inBuff.size = inBuff.pos + 1; outBuff.size = outBuff.pos + 1; r = ZSTD_decompressStream(zd, &outBuff, &inBuff); @@ -322,6 +324,8 @@ _output_error: } +/* ====== Fuzzer tests ====== */ + static size_t findDiff(const void* buf1, const void* buf2, size_t max) { const BYTE* b1 = (const BYTE*)buf1; @@ -413,8 +417,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; - /* states full reset (unsynchronized) */ - /* some issues only happen when reusing states in a specific sequence of parameters */ + /* states full reset (deliberately not synchronized) */ + /* some issues can only happen when reusing states */ if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); } if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); } diff --git a/zstd_compression_format.md b/zstd_compression_format.md index b14f55534..b58b43f5a 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -16,7 +16,7 @@ Distribution of this document is unlimited. ### Version -0.2.0 (22/07/16) +0.2.2 (14/09/16) Introduction @@ -1049,15 +1049,23 @@ by reading the required `Number_of_Bits`, and adding the specified `Baseline`. #### Bitstream -All sequences are stored in a single bitstream, read _backward_. -It is therefore necessary to know the bitstream size, -which is deducted from compressed block size. +FSE bitstreams are read in reverse direction than written. In zstd, +the compressor writes bits forward into a block and the decompressor +must read the bitstream _backwards_. -The last useful bit of the stream is followed by an end-bit-flag. -Highest bit of last byte is this flag. -It does not belong to the useful part of the bitstream. -Therefore, last byte has 0-7 useful bits. -Note that it also means that last byte cannot be `0`. +To find the start of the bitstream it is therefore necessary to +know the offset of the last byte of the block which can be found +by counting `Block_Size` bytes after the block header. + +After writing the last bit containing information, the compressor +writes a single `1`-bit and then fills the byte with 0-7 `0` bits of +padding. The last byte of the compressed bitstream cannot be `0` for +that reason. + +When decompressing, the last byte containing the padding is the first +byte to read. The decompressor needs to skip 0-7 initial `0`-bits and +the first `1`-bit it occurs. Afterwards, the useful part of the bitstream +begins. ##### Starting states @@ -1164,9 +1172,192 @@ __`Content`__ : The rest of the dictionary is its content. [compressed blocks]: #the-format-of-compressed_block +Appendix A - Decoding tables for predefined codes +------------------------------------------------- + +This appendix contains FSE decoding tables for the predefined literal length, match length, and offset +codes. The tables have been constructed using the algorithm as given above in the +"from normalized distribution to decoding tables" chapter. The tables here can be used as examples +to crosscheck that an implementation implements the decoding table generation algorithm correctly. + +#### Literal Length Code: + +| State | Symbol | Number_Of_Bits | Base | +| ----- | ------ | -------------- | ---- | +| 0 | 0 | 4 | 0 | +| 1 | 0 | 4 | 16 | +| 2 | 1 | 5 | 32 | +| 3 | 3 | 5 | 0 | +| 4 | 4 | 5 | 0 | +| 5 | 6 | 5 | 0 | +| 6 | 7 | 5 | 0 | +| 7 | 9 | 5 | 0 | +| 8 | 10 | 5 | 0 | +| 9 | 12 | 5 | 0 | +| 10 | 14 | 6 | 0 | +| 11 | 16 | 5 | 0 | +| 12 | 18 | 5 | 0 | +| 13 | 19 | 5 | 0 | +| 14 | 21 | 5 | 0 | +| 15 | 22 | 5 | 0 | +| 16 | 24 | 5 | 0 | +| 17 | 25 | 5 | 32 | +| 18 | 26 | 5 | 0 | +| 19 | 27 | 6 | 0 | +| 20 | 29 | 6 | 0 | +| 21 | 31 | 6 | 0 | +| 22 | 0 | 4 | 32 | +| 23 | 1 | 4 | 0 | +| 24 | 2 | 5 | 0 | +| 25 | 4 | 5 | 32 | +| 26 | 5 | 5 | 0 | +| 27 | 7 | 5 | 32 | +| 28 | 8 | 5 | 0 | +| 29 | 10 | 5 | 32 | +| 30 | 11 | 5 | 0 | +| 31 | 13 | 6 | 0 | +| 32 | 16 | 5 | 32 | +| 33 | 17 | 5 | 0 | +| 34 | 19 | 5 | 32 | +| 35 | 20 | 5 | 0 | +| 36 | 22 | 5 | 32 | +| 37 | 23 | 5 | 0 | +| 38 | 25 | 4 | 0 | +| 39 | 25 | 4 | 16 | +| 40 | 26 | 5 | 32 | +| 41 | 28 | 6 | 0 | +| 42 | 30 | 6 | 0 | +| 43 | 0 | 4 | 48 | +| 44 | 1 | 4 | 16 | +| 45 | 2 | 5 | 32 | +| 46 | 3 | 5 | 32 | +| 47 | 5 | 5 | 32 | +| 48 | 6 | 5 | 32 | +| 49 | 8 | 5 | 32 | +| 50 | 9 | 5 | 32 | +| 51 | 11 | 5 | 32 | +| 52 | 12 | 5 | 32 | +| 53 | 15 | 6 | 0 | +| 54 | 17 | 5 | 32 | +| 55 | 18 | 5 | 32 | +| 56 | 20 | 5 | 32 | +| 57 | 21 | 5 | 32 | +| 58 | 23 | 5 | 32 | +| 59 | 24 | 5 | 32 | +| 60 | 35 | 6 | 0 | +| 61 | 34 | 6 | 0 | +| 62 | 33 | 6 | 0 | +| 63 | 32 | 6 | 0 | + +#### Match Length Code: + +| State | Symbol | Number_Of_Bits | Base | +| ----- | ------ | -------------- | ---- | +| 0 | 0 | 6 | 0 | +| 1 | 1 | 4 | 0 | +| 2 | 2 | 5 | 32 | +| 3 | 3 | 5 | 0 | +| 4 | 5 | 5 | 0 | +| 5 | 6 | 5 | 0 | +| 6 | 8 | 5 | 0 | +| 7 | 10 | 6 | 0 | +| 8 | 13 | 6 | 0 | +| 9 | 16 | 6 | 0 | +| 10 | 19 | 6 | 0 | +| 11 | 22 | 6 | 0 | +| 12 | 25 | 6 | 0 | +| 13 | 28 | 6 | 0 | +| 14 | 31 | 6 | 0 | +| 15 | 33 | 6 | 0 | +| 16 | 35 | 6 | 0 | +| 17 | 37 | 6 | 0 | +| 18 | 39 | 6 | 0 | +| 19 | 41 | 6 | 0 | +| 20 | 43 | 6 | 0 | +| 21 | 45 | 6 | 0 | +| 22 | 1 | 4 | 16 | +| 23 | 2 | 4 | 0 | +| 24 | 3 | 5 | 32 | +| 25 | 4 | 5 | 0 | +| 26 | 6 | 5 | 32 | +| 27 | 7 | 5 | 0 | +| 28 | 9 | 6 | 0 | +| 29 | 12 | 6 | 0 | +| 30 | 15 | 6 | 0 | +| 31 | 18 | 6 | 0 | +| 32 | 21 | 6 | 0 | +| 33 | 24 | 6 | 0 | +| 34 | 27 | 6 | 0 | +| 35 | 30 | 6 | 0 | +| 36 | 32 | 6 | 0 | +| 37 | 34 | 6 | 0 | +| 38 | 36 | 6 | 0 | +| 39 | 38 | 6 | 0 | +| 40 | 40 | 6 | 0 | +| 41 | 42 | 6 | 0 | +| 42 | 44 | 6 | 0 | +| 43 | 1 | 4 | 32 | +| 44 | 1 | 4 | 48 | +| 45 | 2 | 4 | 16 | +| 46 | 4 | 5 | 32 | +| 47 | 5 | 5 | 32 | +| 48 | 7 | 5 | 32 | +| 49 | 8 | 5 | 32 | +| 50 | 11 | 6 | 0 | +| 51 | 14 | 6 | 0 | +| 52 | 17 | 6 | 0 | +| 53 | 20 | 6 | 0 | +| 54 | 23 | 6 | 0 | +| 55 | 26 | 6 | 0 | +| 56 | 29 | 6 | 0 | +| 57 | 52 | 6 | 0 | +| 58 | 51 | 6 | 0 | +| 59 | 50 | 6 | 0 | +| 60 | 49 | 6 | 0 | +| 61 | 48 | 6 | 0 | +| 62 | 47 | 6 | 0 | +| 63 | 46 | 6 | 0 | + +#### Offset Code: + +| State | Symbol | Number_Of_Bits | Base | +| ----- | ------ | -------------- | ---- | +| 0 | 0 | 5 | 0 | +| 1 | 6 | 4 | 0 | +| 2 | 9 | 5 | 0 | +| 3 | 15 | 5 | 0 | +| 4 | 21 | 5 | 0 | +| 5 | 3 | 5 | 0 | +| 6 | 7 | 4 | 0 | +| 7 | 12 | 5 | 0 | +| 8 | 18 | 5 | 0 | +| 9 | 23 | 5 | 0 | +| 10 | 5 | 5 | 0 | +| 11 | 8 | 4 | 0 | +| 12 | 14 | 5 | 0 | +| 13 | 20 | 5 | 0 | +| 14 | 2 | 5 | 0 | +| 15 | 7 | 4 | 16 | +| 16 | 11 | 5 | 0 | +| 17 | 17 | 5 | 0 | +| 18 | 22 | 5 | 0 | +| 19 | 4 | 5 | 0 | +| 20 | 8 | 4 | 16 | +| 21 | 13 | 5 | 0 | +| 22 | 19 | 5 | 0 | +| 23 | 1 | 5 | 0 | +| 24 | 6 | 4 | 16 | +| 25 | 10 | 5 | 0 | +| 26 | 16 | 5 | 0 | +| 27 | 28 | 5 | 0 | +| 28 | 27 | 5 | 0 | +| 29 | 26 | 5 | 0 | +| 30 | 25 | 5 | 0 | +| 31 | 24 | 5 | 0 | Version changes --------------- +- 0.2.2 : added predefined codes, by Johannes Rudolph - 0.2.1 : clarify field names, by Przemyslaw Skibinski - 0.2.0 : numerous format adjustments for zstd v0.8 - 0.1.2 : limit Huffman tree depth to 11 bits