Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev

NEWS

@@ -4,8 +4,10 @@ added : NetBSD install target (#338)
 Improved : variable compression speed improvements on batches of small files.
 Fixed : CLI -d output to stdout by default when input is stdin (#322)
 Fixed : CLI correctly detects console on Mac OS-X
-Fixed : Legacy decoders use unified error codes (#341), reported by benrg
+Fixed : CLI supports recursive mode `-r` on Mac OS-X
+Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
 Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
+Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
 Fixed : zstd-pgo, reported by octoploid (#329)
 
 v1.0.0

lib/compress/zstd_compress.c

@@ -113,6 +113,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
 
 size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
 {
+    if (cctx==NULL) return 0;   /* support sizeof on NULL */
     return sizeof(*cctx) + cctx->workSpaceSize;
 }
 
@@ -168,7 +169,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
     {   U32 const minSrcSize = (srcSize==0) ? 500 : 0;
         U64 const rSize = srcSize + dictSize + minSrcSize;
         if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
-            U32 const srcLog = ZSTD_highbit32((U32)(rSize)-1) + 1;
+            U32 const srcLog = MAX(4, ZSTD_highbit32((U32)(rSize)-1) + 1);
             if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
     }   }
     if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
@@ -238,9 +239,9 @@ typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_comp
     note : 'params' must be validated */
 static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
                                        ZSTD_parameters params, U64 frameContentSize,
-                                       ZSTD_compResetPolicy_e crp)
+                                       ZSTD_compResetPolicy_e const crp)
 {
-    if (crp == ZSTDcrp_continue)   /* still some issues */
+    if (crp == ZSTDcrp_continue)
         if (ZSTD_equivalentParams(params, zc->params))
             return ZSTD_continueCCtx(zc, params, frameContentSize);
 
@@ -2680,6 +2681,12 @@ struct ZSTD_CDict_s {
     ZSTD_CCtx* refContext;
 };  /* typedef'd tp ZSTD_CDict within "zstd.h" */
 
+size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
+{
+    if (cdict==NULL) return 0;   /* support sizeof on NULL */
+    return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize;
+}
+
 ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
 {
     if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
@@ -2731,17 +2738,23 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
     }
 }
 
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
+{
+    if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext))
+    else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
+    return 0;
+}
+
 /*! ZSTD_compress_usingCDict() :
 *   Compression using a digested Dictionary.
 *   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
 *   Note that compression level is decided during dictionary creation */
-ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
+size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
-                                           void* dst, size_t dstCapacity,
+                                void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
+                                const void* src, size_t srcSize,
-                                     const ZSTD_CDict* cdict)
+                                const ZSTD_CDict* cdict)
 {
-    if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext))
+    CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));
-    else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, srcSize));
 
     if (cdict->refContext->params.fParams.contentSizeFlag==1) {
         cctx->params.fParams.contentSizeFlag = 1;
@@ -2760,7 +2773,8 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
 typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
 
 struct ZSTD_CStream_s {
-    ZSTD_CCtx* zc;
+    ZSTD_CCtx* cctx;
+    ZSTD_CDict* cdict;
     char*  inBuff;
     size_t inBuffSize;
     size_t inToCompress;
@@ -2793,8 +2807,8 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
     if (zcs==NULL) return NULL;
     memset(zcs, 0, sizeof(ZSTD_CStream));
     memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
-    zcs->zc = ZSTD_createCCtx_advanced(customMem);
+    zcs->cctx = ZSTD_createCCtx_advanced(customMem);
-    if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; }
+    if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; }
     return zcs;
 }
 
@@ -2802,7 +2816,8 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
 {
     if (zcs==NULL) return 0;   /* support free on NULL */
     {   ZSTD_customMem const cMem = zcs->customMem;
-        ZSTD_freeCCtx(zcs->zc);
+        ZSTD_freeCCtx(zcs->cctx);
+        ZSTD_freeCDict(zcs->cdict);
         ZSTD_free(zcs->inBuff, cMem);
         ZSTD_free(zcs->outBuff, cMem);
         ZSTD_free(zcs, cMem);
@@ -2816,6 +2831,19 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
 size_t ZSTD_CStreamInSize(void)  { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
 size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
 
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
+{
+    CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize));
+
+    zcs->inToCompress = 0;
+    zcs->inBuffPos = 0;
+    zcs->inBuffTarget = zcs->blockSize;
+    zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
+    zcs->stage = zcss_load;
+    zcs->frameEnded = 0;
+    return 0;   /* ready to go */
+}
+
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  const void* dict, size_t dictSize,
                                  ZSTD_parameters params, unsigned long long pledgedSrcSize)
@@ -2837,16 +2865,13 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
         if (zcs->outBuff == NULL) return ERROR(memory_allocation);
     }
 
-    CHECK_F(ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize));
+    ZSTD_freeCDict(zcs->cdict);
+    zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
+    if (zcs->cdict == NULL) return ERROR(memory_allocation);
 
-    zcs->inToCompress = 0;
-    zcs->inBuffPos = 0;
-    zcs->inBuffTarget = zcs->blockSize;
-    zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
-    zcs->stage = zcss_load;
     zcs->checksum = params.fParams.checksumFlag > 0;
-    zcs->frameEnded = 0;
+
-    return 0;   /* ready to go */
+    return ZSTD_resetCStream(zcs, pledgedSrcSize);
 }
 
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
@@ -2862,7 +2887,8 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
 {
-    return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize;
+    if (zcs==NULL) return 0;   /* support sizeof on NULL */
+    return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize;
 }
 
 /*======   Compression   ======*/
@@ -2913,8 +2939,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 else
                     cDst = zcs->outBuff, oSize = zcs->outBuffSize;
                 cSize = (flush == zsf_end) ?
-                        ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
+                        ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
-                        ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
+                        ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
                 if (ZSTD_isError(cSize)) return cSize;
                 if (flush == zsf_end) zcs->frameEnded = 1;
                 /* prepare next block */
@@ -3008,7 +3034,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
         /* create epilogue */
         zcs->stage = zcss_final;
         zcs->outBuffContentSize = !notEnded ? 0 :
-            ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0);  /* write epilogue, including final empty block, into outBuff */
+            ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0);  /* write epilogue, including final empty block, into outBuff */
     }
 
     /* flush epilogue */

lib/decompress/zstd_decompress.c

@@ -80,8 +80,12 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
 
 struct ZSTD_DCtx_s
 {
+    const FSE_DTable* LLTptr;
+    const FSE_DTable* MLTptr;
+    const FSE_DTable* OFTptr;
+    const HUF_DTable* HUFptr;
     FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
-    FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+    FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
     FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
     HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)];  /* can accommodate HUF_decompress4X */
     const void* previousDstEnd;
@@ -107,7 +111,7 @@ struct ZSTD_DCtx_s
     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
 };  /* typedef'd to ZSTD_DCtx within "zstd.h" */
 
-size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); }
+size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; return sizeof(ZSTD_DCtx); }  /* support sizeof on NULL */
 
 size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
 
@@ -119,11 +123,15 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
     dctx->base = NULL;
     dctx->vBase = NULL;
     dctx->dictEnd = NULL;
-    dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);
+    dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
     dctx->litEntropy = dctx->fseEntropy = 0;
     dctx->dictID = 0;
     MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue));
-    memcpy(dctx->rep, repStartValue, sizeof(repStartValue));
+    memcpy(dctx->rep, repStartValue, sizeof(repStartValue));  /* initial repcodes */
+    dctx->LLTptr = dctx->LLTable;
+    dctx->MLTptr = dctx->MLTable;
+    dctx->OFTptr = dctx->OFTable;
+    dctx->HUFptr = dctx->hufTable;
     return 0;
 }
 
@@ -159,6 +167,25 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
     memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize);  /* no need to copy workspace */
 }
 
+static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
+{
+    ZSTD_decompressBegin(dstDCtx);  /* init */
+    dstDCtx->dictEnd = srcDCtx->dictEnd;
+    dstDCtx->vBase = srcDCtx->vBase;
+    dstDCtx->base = srcDCtx->base;
+    dstDCtx->previousDstEnd = srcDCtx->previousDstEnd;
+    dstDCtx->dictID = srcDCtx->dictID;
+    dstDCtx->litEntropy = srcDCtx->litEntropy;
+    dstDCtx->fseEntropy = srcDCtx->fseEntropy;
+    dstDCtx->LLTptr = srcDCtx->LLTable;
+    dstDCtx->MLTptr = srcDCtx->MLTable;
+    dstDCtx->OFTptr = srcDCtx->OFTable;
+    dstDCtx->HUFptr = srcDCtx->hufTable;
+    dstDCtx->rep[0] = srcDCtx->rep[0];
+    dstDCtx->rep[1] = srcDCtx->rep[1];
+    dstDCtx->rep[2] = srcDCtx->rep[2];
+}
+
 
 /*-*************************************************************
 *   Decompression section
@@ -350,34 +377,31 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 {
                 case 0: case 1: default:   /* note : default is impossible, since lhlCode into [0..3] */
                     /* 2 - 2 - 10 - 10 */
-                    {   singleStream = !lhlCode;
+                    singleStream = !lhlCode;
-                        lhSize = 3;
+                    lhSize = 3;
-                        litSize  = (lhc >> 4) & 0x3FF;
+                    litSize  = (lhc >> 4) & 0x3FF;
-                        litCSize = (lhc >> 14) & 0x3FF;
+                    litCSize = (lhc >> 14) & 0x3FF;
-                        break;
+                    break;
-                    }
                 case 2:
                     /* 2 - 2 - 14 - 14 */
-                    {   lhSize = 4;
+                    lhSize = 4;
-                        litSize  = (lhc >> 4) & 0x3FFF;
+                    litSize  = (lhc >> 4) & 0x3FFF;
-                        litCSize = lhc >> 18;
+                    litCSize = lhc >> 18;
-                        break;
+                    break;
-                    }
                 case 3:
                     /* 2 - 2 - 18 - 18 */
-                    {   lhSize = 5;
+                    lhSize = 5;
-                        litSize  = (lhc >> 4) & 0x3FFFF;
+                    litSize  = (lhc >> 4) & 0x3FFFF;
-                        litCSize = (lhc >> 22) + (istart[4] << 10);
+                    litCSize = (lhc >> 22) + (istart[4] << 10);
-                        break;
+                    break;
-                    }
                 }
                 if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
                 if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
 
                 if (HUF_isError((litEncType==set_repeat) ?
                                     ( singleStream ?
-                                        HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) :
+                                        HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) :
-                                        HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) :
+                                        HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) :
                                     ( singleStream ?
                                         HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
                                         HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
@@ -387,6 +411,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
                 dctx->litSize = litSize;
                 dctx->litEntropy = 1;
+                if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable;
                 return litCSize + lhSize;
             }
 
@@ -461,7 +486,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
     @return : nb bytes read from src,
               or an error code if it fails, testable with ZSTD_isError()
 */
-FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog,
+static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog,
                                  const void* src, size_t srcSize,
                                  const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
 {
@@ -491,8 +516,7 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e
 }
 
 
-size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
+size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
-                             FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 flagRepeatTable,
                              const void* src, size_t srcSize)
 {
     const BYTE* const istart = (const BYTE* const)src;
@@ -522,16 +546,19 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
         ip++;
 
         /* Build DTables */
-        {   size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
+        {   size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, dctx->fseEntropy);
             if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
+            if (LLtype != set_repeat) dctx->LLTptr = dctx->LLTable;
             ip += llhSize;
         }
-        {   size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
+        {   size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, dctx->fseEntropy);
             if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
+            if (OFtype != set_repeat) dctx->OFTptr = dctx->OFTable;
             ip += ofhSize;
         }
-        {   size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
+        {   size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, dctx->fseEntropy);
             if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
+            if (MLtype != set_repeat) dctx->MLTptr = dctx->MLTable;
             ip += mlhSize;
     }   }
 
@@ -714,16 +741,13 @@ static size_t ZSTD_decompressSequences(
     const BYTE* litPtr = dctx->litPtr;
     const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
     const BYTE* const litEnd = litPtr + dctx->litSize;
-    FSE_DTable* DTableLL = dctx->LLTable;
-    FSE_DTable* DTableML = dctx->MLTable;
-    FSE_DTable* DTableOffb = dctx->OffTable;
     const BYTE* const base = (const BYTE*) (dctx->base);
     const BYTE* const vBase = (const BYTE*) (dctx->vBase);
     const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
     int nbSeq;
 
     /* Build Decoding Tables */
-    {   size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize);
+    {   size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
         if (ZSTD_isError(seqHSize)) return seqHSize;
         ip += seqHSize;
     }
@@ -733,10 +757,10 @@ static size_t ZSTD_decompressSequences(
         seqState_t seqState;
         dctx->fseEntropy = 1;
         { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
-        CHECK_E(BIT_initDStream(&(seqState.DStream), ip, iend-ip), corruption_detected);
+        CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
-        FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
+        FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
-        FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
+        FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
-        FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
+        FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
 
         for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
             nbSeq--;
@@ -894,25 +918,10 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
 }
 
 
-/*! ZSTD_decompress_usingPreparedDCtx() :
-*   Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
-*   It avoids reloading the dictionary each time.
-*   `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
-*   Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
-size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx,
-                                         void* dst, size_t dstCapacity,
-                                   const void* src, size_t srcSize)
-{
-    ZSTD_copyDCtx(dctx, refDCtx);
-    ZSTD_checkContinuity(dctx, dst);
-    return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
-}
-
-
 size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                  void* dst, size_t dstCapacity,
-                                 const void* src, size_t srcSize,
+                           const void* src, size_t srcSize,
-                                 const void* dict, size_t dictSize)
+                           const void* dict, size_t dictSize)
 {
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
     if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
@@ -1120,7 +1129,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
         size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
         dictPtr += offcodeHeaderSize;
     }
 
@@ -1172,7 +1181,6 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
     return ZSTD_refDictContent(dctx, dict, dictSize);
 }
 
-
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
     CHECK_F(ZSTD_decompressBegin(dctx));
@@ -1181,6 +1189,8 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
 }
 
 
+/* ======   ZSTD_DDict   ====== */
+
 struct ZSTD_DDict_s {
     void* dict;
     size_t dictSize;
@@ -1239,20 +1249,27 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
     }
 }
 
+size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
+{
+    if (ddict==NULL) return 0;   /* support sizeof on NULL */
+    return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize;
+}
+
+
 /*! ZSTD_decompress_usingDDict() :
 *   Decompression using a pre-digested Dictionary
 *   Use dictionary without significant overhead. */
-ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
+size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
-                                           void* dst, size_t dstCapacity,
+                                  void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
+                            const void* src, size_t srcSize,
-                                     const ZSTD_DDict* ddict)
+                            const ZSTD_DDict* ddict)
 {
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
     if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
 #endif
-    return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext,
+    ZSTD_refDCtx(dctx, ddict->refContext);
-                                             dst, dstCapacity,
+    ZSTD_checkContinuity(dctx, dst);
-                                             src, srcSize);
+    return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
 }
 
 
@@ -1265,7 +1282,8 @@ typedef enum { zdss_init, zdss_loadHeader,
 
 /* *** Resource management *** */
 struct ZSTD_DStream_s {
-    ZSTD_DCtx* zd;
+    ZSTD_DCtx* dctx;
+    ZSTD_DDict* ddict;
     ZSTD_frameParams fParams;
     ZSTD_dStreamStage stage;
     char*  inBuff;
@@ -1277,12 +1295,9 @@ struct ZSTD_DStream_s {
     size_t outStart;
     size_t outEnd;
     size_t blockSize;
-    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];   /* tmp buffer to store frame header */
     size_t lhSize;
     ZSTD_customMem customMem;
-    void* dictContent;
-    size_t dictSize;
-    const void* dictSource;
     void* legacyContext;
     U32 previousLegacyVersion;
     U32 legacyVersion;
@@ -1306,8 +1321,8 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
     if (zds==NULL) return NULL;
     memset(zds, 0, sizeof(ZSTD_DStream));
     memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
-    zds->zd = ZSTD_createDCtx_advanced(customMem);
+    zds->dctx = ZSTD_createDCtx_advanced(customMem);
-    if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; }
+    if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; }
     zds->stage = zdss_init;
     zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
     return zds;
@@ -1317,10 +1332,10 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
 {
     if (zds==NULL) return 0;   /* support free on null */
     {   ZSTD_customMem const cMem = zds->customMem;
-        ZSTD_freeDCtx(zds->zd);
+        ZSTD_freeDCtx(zds->dctx);
+        ZSTD_freeDDict(zds->ddict);
         ZSTD_free(zds->inBuff, cMem);
         ZSTD_free(zds->outBuff, cMem);
-        ZSTD_free(zds->dictContent, cMem);
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
         if (zds->legacyContext)
             ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion);
@@ -1340,15 +1355,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
 {
     zds->stage = zdss_loadHeader;
     zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
-    if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) {   /* new dictionary */
+    ZSTD_freeDDict(zds->ddict);
-        if (dictSize > zds->dictSize) {
+    zds->ddict = ZSTD_createDDict(dict, dictSize);
-            ZSTD_free(zds->dictContent, zds->customMem);
+    if (zds->ddict == NULL) return ERROR(memory_allocation);
-            zds->dictContent = ZSTD_malloc(dictSize, zds->customMem);
-            if (zds->dictContent == NULL) return ERROR(memory_allocation);
-        }
-        memcpy(zds->dictContent, dict, dictSize);
-        zds->dictSize = dictSize;
-    }
     zds->legacyVersion = 0;
     zds->hostageByte = 0;
     return ZSTD_frameHeaderSize_prefix;
@@ -1359,6 +1368,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
     return ZSTD_initDStream_usingDict(zds, NULL, 0);
 }
 
+size_t ZSTD_resetDStream(ZSTD_DStream* zds)
+{
+    zds->stage = zdss_loadHeader;
+    zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
+    zds->legacyVersion = 0;
+    zds->hostageByte = 0;
+    return ZSTD_frameHeaderSize_prefix;
+}
+
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
                                 ZSTD_DStreamParameter_e paramType, unsigned paramValue)
 {
@@ -1373,7 +1391,8 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
 
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
 {
-    return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize + zds->dictSize;
+    if (zds==NULL) return 0;   /* support sizeof on NULL */
+    return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize;
 }
 
 
@@ -1415,7 +1434,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 {   U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
                     if (legacyVersion) {
                         CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
-                                                       zds->dictContent, zds->dictSize));
+                                                       zds->ddict->dict, zds->ddict->dictSize));
                         zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
                         return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
                     } else {
@@ -1437,11 +1456,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             /* Consume header */
-            ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize);
+            ZSTD_refDCtx(zds->dctx, zds->ddict->refContext);
-            {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd);  /* == ZSTD_frameHeaderSize_prefix */
+            {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);  /* == ZSTD_frameHeaderSize_prefix */
-                CHECK_F(ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size));
+                CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
-                {   size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd);
+                {   size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
-                    CHECK_F(ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size));
+                    CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size));
             }   }
 
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
@@ -1467,15 +1486,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             /* pass-through */
 
         case zdss_read:
-            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
+            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
                 if (neededInSize==0) {  /* end of frame */
                     zds->stage = zdss_init;
                     someMoreWork = 0;
                     break;
                 }
                 if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
-                    const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
+                    const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
-                    size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
+                    size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
                         zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
                         ip, neededInSize);
                     if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -1491,7 +1510,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }
 
         case zdss_load:
-            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
+            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
                 size_t const toLoad = neededInSize - zds->inPos;   /* should always be <= remaining space within inBuff */
                 size_t loadedSize;
                 if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected);   /* should never happen */
@@ -1501,8 +1520,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */
 
                 /* decode loaded input */
-                {  const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
+                {  const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
-                   size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
+                   size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
                         zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
                         zds->inBuff, neededInSize);
                     if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -1534,7 +1553,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     /* result */
     input->pos += (size_t)(ip-istart);
     output->pos += (size_t)(op-ostart);
-    {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
+    {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
         if (!nextSrcSizeHint) {   /* frame fully decoded */
             if (zds->outEnd == zds->outStart) {  /* output fully flushed */
                 if (zds->hostageByte) {
@@ -1549,7 +1568,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }
             return 1;
         }
-        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);   /* preload header of next block */
+        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block);   /* preload header of next block */
         if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC);   /* should never happen */
         nextSrcSizeHint -= zds->inPos;   /* already loaded*/
         return nextSrcSizeHint;

lib/zstd.h

@@ -48,40 +48,41 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void);
 *  Simple API
 ***************************************/
 /*! ZSTD_compress() :
-    Compresses `src` buffer into already allocated `dst`.
+    Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
     Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
-    @return : the number of bytes written into `dst` (<= `dstCapacity),
+    @return : compressed size written into `dst` (<= `dstCapacity),
               or an error code if it fails (which can be tested using ZSTD_isError()) */
 ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
                             const void* src, size_t srcSize,
                                   int compressionLevel);
 
-/*! ZSTD_getDecompressedSize() :
-*   @return : decompressed size as a 64-bits value _if known_, 0 otherwise.
-*    note 1 : decompressed size can be very large (64-bits value),
-*             potentially larger than what local system can handle as a single memory segment.
-*             In which case, it's necessary to use streaming mode to decompress data.
-*    note 2 : decompressed size is an optional field, that may not be present.
-*             When `return==0`, data to decompress can have any size.
-*             In which case, it's necessary to use streaming mode to decompress data.
-*             Optionally, application may rely on its own implied limits.
-*             (For example, application data could be necessarily cut into blocks <= 16 KB).
-*    note 3 : decompressed size could be wrong or intentionally modified !
-*             Always ensure result fits within application's authorized limits !
-*             Each application can set its own limits.
-*    note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
-ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
-
 /*! ZSTD_decompress() :
-    `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail.
+    `compressedSize` : must be the _exact_ size of a single compressed frame.
-    `dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ).
+    `dstCapacity` is an upper bound of originalSize.
-    If originalSize is unknown, and if there is no implied application-specific limitations,
+    If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
-    it's preferable to use streaming mode to decompress data.
     @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
               or an errorCode if it fails (which can be tested using ZSTD_isError()) */
 ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                               const void* src, size_t compressedSize);
 
+/*! ZSTD_getDecompressedSize() :
+*   'src' is the start of a zstd compressed frame.
+*   @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
+*    note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+*             When `return==0`, data to decompress could be any size.
+*             In which case, it's necessary to use streaming mode to decompress data.
+*             Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+*             (For example, data may be necessarily cut into blocks <= 16 KB).
+*    note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+*    note 3 : decompressed size can be very large (64-bits value),
+*             potentially larger than what local system can handle as a single memory segment.
+*             In which case, it's necessary to use streaming mode to decompress data.
+*    note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+*             Always ensure result fits within application's authorized limits.
+*             Each application can set its own limits.
+*    note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
+ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+
 
 /*======  Helper functions  ======*/
 ZSTDLIB_API int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
@@ -351,14 +352,18 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
  *  Create a ZSTD compression context using external alloc and free functions */
 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 
+/*! ZSTD_sizeofCCtx() :
+ *  Gives the amount of memory used by a given ZSTD_CCtx */
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+
 /*! ZSTD_createCDict_advanced() :
  *  Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
                                                   ZSTD_parameters params, ZSTD_customMem customMem);
 
-/*! ZSTD_sizeofCCtx() :
+/*! ZSTD_sizeof_CDict() :
- *  Gives the amount of memory used by a given ZSTD_CCtx */
+ *  Gives the amount of memory used by a given ZSTD_sizeof_CDict */
-ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
 
 /*! ZSTD_getParams() :
 *   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
@@ -398,10 +403,14 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
  *  Create a ZSTD decompression context using external alloc and free functions */
 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
 
-/*! ZSTD_sizeofDCtx() :
+/*! ZSTD_sizeof_DCtx() :
  *  Gives the amount of memory used by a given ZSTD_DCtx */
 ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
 
+/*! ZSTD_sizeof_DDict() :
+ *  Gives the amount of memory used by a given ZSTD_DDict */
+ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
 
 /* ******************************************************************
 *  Advanced Streaming functions
@@ -412,7 +421,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                 ZSTD_parameters params, unsigned long long pledgedSrcSize);
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; saves dictionary loading */
 ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
 
 
@@ -423,6 +433,7 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
 ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 
 

programs/datagen.c

@@ -20,10 +20,9 @@
 /*-************************************
 *  Dependencies
 **************************************/
-#include <stdlib.h>    /* malloc */
+#include <stdlib.h>    /* malloc, free */
 #include <stdio.h>     /* FILE, fwrite, fprintf */
 #include <string.h>    /* memcpy */
-#include <errno.h>     /* errno */
 #include "mem.h"       /* U32 */
 
 
@@ -176,7 +175,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
     BYTE ldt[LTSIZE];   /* literals distribution table */
 
     /* init */
-    if (buff==NULL) { fprintf(stderr, "datagen: error: %s \n", strerror(errno)); exit(1); }
+    if (buff==NULL) { perror("datagen"); exit(1); }
     if (litProba<=0.0) litProba = matchProba / 4.5;
     memset(ldt, '0', sizeof(ldt));   /* yes, character '0', this is intentional */
     RDG_fillLiteralDistrib(ldt, litProba);

programs/fileio.c

@@ -119,8 +119,6 @@ static clock_t g_time = 0;
 ***************************************/
 static U32 g_overwrite = 0;
 void FIO_overwriteMode(void) { g_overwrite=1; }
-static U32 g_maxWLog = 23;
-void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
 static U32 g_sparseFileSupport = 1;   /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
 void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
 static U32 g_dictIDFlag = 1;
@@ -167,7 +165,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
     return f;
 }
 
-/* `dstFileName must` be non-NULL */
+/** FIO_openDstFile() :
+ * condition : `dstFileName` must be non-NULL.
+ * @result : FILE* to `dstFileName`, or NULL if it fails */
 static FILE* FIO_openDstFile(const char* dstFileName)
 {
     FILE* f;
@@ -250,14 +250,12 @@ typedef struct {
     size_t srcBufferSize;
     void*  dstBuffer;
     size_t dstBufferSize;
-    void*  dictBuffer;
-    size_t dictBufferSize;
     ZSTD_CStream* cctx;
     FILE* dstFile;
     FILE* srcFile;
 } cRess_t;
 
-static cRess_t FIO_createCResources(const char* dictFileName)
+static cRess_t FIO_createCResources(const char* dictFileName, int cLevel)
 {
     cRess_t ress;
     memset(&ress, 0, sizeof(ress));
@@ -271,19 +269,27 @@ static cRess_t FIO_createCResources(const char* dictFileName)
     if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory");
 
     /* dictionary */
-    ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName);
+    {   void* dictBuffer;
+        size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName);
+        if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
+        {   ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictBuffSize);
+            params.fParams.contentSizeFlag = 1;
+            params.fParams.checksumFlag = g_checksumFlag;
+            params.fParams.noDictIDFlag = !g_dictIDFlag;
+            {   size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, 0);
+                if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
+        }   }
+        free(dictBuffer);
+    }
 
     return ress;
 }
 
 static void FIO_freeCResources(cRess_t ress)
 {
-    size_t errorCode;
     free(ress.srcBuffer);
     free(ress.dstBuffer);
-    free(ress.dictBuffer);
+    ZSTD_freeCStream(ress.cctx);   /* never fails */
-    errorCode = ZSTD_freeCStream(ress.cctx);
-    if (ZSTD_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZSTD_CStream : %s", ZSTD_getErrorName(errorCode));
 }
 
 
@@ -293,8 +299,7 @@ static void FIO_freeCResources(cRess_t ress)
  *            1 : missing or pb opening srcFileName
  */
 static int FIO_compressFilename_internal(cRess_t ress,
-                                         const char* dstFileName, const char* srcFileName,
+                                         const char* dstFileName, const char* srcFileName)
-                                         int cLevel)
 {
     FILE* const srcFile = ress.srcFile;
     FILE* const dstFile = ress.dstFile;
@@ -303,17 +308,9 @@ static int FIO_compressFilename_internal(cRess_t ress,
     U64 const fileSize = UTIL_getFileSize(srcFileName);
 
     /* init */
-    {   ZSTD_parameters params = ZSTD_getParams(cLevel, fileSize, ress.dictBufferSize);
+    {   size_t const resetError = ZSTD_resetCStream(ress.cctx, fileSize);
-        params.fParams.contentSizeFlag = 1;
+        if (ZSTD_isError(resetError)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(resetError));
-        params.fParams.checksumFlag = g_checksumFlag;
+    }
-        params.fParams.noDictIDFlag = !g_dictIDFlag;
-        if ((g_maxWLog) && (params.cParams.windowLog > g_maxWLog)) {
-            params.cParams.windowLog = g_maxWLog;
-            params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize);
-        }
-        {   size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
-            if (ZSTD_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(errorCode));
-    }   }
 
     /* Main compression loop */
     while (1) {
@@ -367,8 +364,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
  *            1 : missing or pb opening srcFileName
  */
 static int FIO_compressFilename_srcFile(cRess_t ress,
-                                        const char* dstFileName, const char* srcFileName,
+                                        const char* dstFileName, const char* srcFileName)
-                                        int cLevel)
 {
     int result;
 
@@ -380,10 +376,10 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (!ress.srcFile) return 1;   /* srcFile could not be opened */
 
-    result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
+    result = FIO_compressFilename_internal(ress, dstFileName, srcFileName);
 
     fclose(ress.srcFile);
-    if ((g_removeSrcFile) && (!result)) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); }
+    if (g_removeSrcFile && !result) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */
     return result;
 }
 
@@ -393,17 +389,16 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
  *            1 : pb
  */
 static int FIO_compressFilename_dstFile(cRess_t ress,
-                                        const char* dstFileName, const char* srcFileName,
+                                        const char* dstFileName, const char* srcFileName)
-                                        int cLevel)
 {
     int result;
 
     ress.dstFile = FIO_openDstFile(dstFileName);
-    if (ress.dstFile==0) return 1;
+    if (ress.dstFile==NULL) return 1;  /* could not open dstFileName */
 
-    result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel);
+    result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName);
 
-    if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; }
+    if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; }  /* error closing dstFile */
     if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); }  /* remove operation artefact */
     return result;
 }
@@ -414,8 +409,8 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
 {
     clock_t const start = clock();
 
-    cRess_t const ress = FIO_createCResources(dictFileName);
+    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel);
-    int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
+    int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName);
 
     double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
     DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
@@ -433,7 +428,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
     size_t dfnSize = FNSPACE;
     char*  dstFileName = (char*)malloc(FNSPACE);
     size_t const suffixSize = suffix ? strlen(suffix) : 0;
-    cRess_t ress = FIO_createCResources(dictFileName);
+    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel);
 
     /* init */
     if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
@@ -445,8 +440,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
         ress.dstFile = stdout;
         SET_BINARY_MODE(stdout);
         for (u=0; u<nbFiles; u++)
-            missed_files += FIO_compressFilename_srcFile(ress, stdoutmark,
+            missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u]);
-                                                         inFileNamesTable[u], compressionLevel);
         if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
     } else {
         unsigned u;
@@ -455,8 +449,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
             if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
             strcpy(dstFileName, inFileNamesTable[u]);
             strcat(dstFileName, suffix);
-            missed_files += FIO_compressFilename_dstFile(ress, dstFileName,
+            missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u]);
-                                                         inFileNamesTable[u], compressionLevel);
     }   }
 
     /* Close & Free */
@@ -480,8 +473,6 @@ typedef struct {
     size_t srcBufferSize;
     void*  dstBuffer;
     size_t dstBufferSize;
-    void*  dictBuffer;
-    size_t dictBufferSize;
     ZSTD_DStream* dctx;
     FILE*  dstFile;
 } dRess_t;
@@ -501,7 +492,12 @@ static dRess_t FIO_createDResources(const char* dictFileName)
     if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
 
     /* dictionary */
-    ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName);
+    {   void* dictBuffer;
+        size_t const dictBufferSize = FIO_loadFile(&dictBuffer, dictFileName);
+        size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize);
+        if (ZSTD_isError(initError)) EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError));
+        free(dictBuffer);
+    }
 
     return ress;
 }
@@ -512,7 +508,6 @@ static void FIO_freeDResources(dRess_t ress)
     if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", ZSTD_getErrorName(errorCode));
     free(ress.srcBuffer);
     free(ress.dstBuffer);
-    free(ress.dictBuffer);
 }
 
 
@@ -601,7 +596,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
     size_t readSize;
     U32 storedSkips = 0;
 
-    ZSTD_initDStream_usingDict(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
+    ZSTD_resetDStream(ress.dctx);
 
     /* Header loading (optional, saves one loop) */
     {   size_t const toLoad = 9 - alreadyLoaded;   /* assumption : 9 >= alreadyLoaded */

programs/fileio.h

@@ -8,7 +8,8 @@
  */
 
 
-#pragma once
+#ifndef FILEIO_H_23981798732
+#define FILEIO_H_23981798732
 
 #if defined (__cplusplus)
 extern "C" {
@@ -32,7 +33,6 @@ extern "C" {
 ***************************************/
 void FIO_overwriteMode(void);
 void FIO_setNotificationLevel(unsigned level);
-void FIO_setMaxWLog(unsigned maxWLog);     /**< if `maxWLog` == 0, no max enforced */
 void FIO_setSparseWrite(unsigned sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
 void FIO_setDictIDFlag(unsigned dictIDFlag);
 void FIO_setChecksumFlag(unsigned checksumFlag);
@@ -70,3 +70,5 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
 #if defined (__cplusplus)
 }
 #endif
+
+#endif  /* FILEIO_H_23981798732 */

programs/util.h

@@ -274,7 +274,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
     return nbFiles;
 }
 
-#elif (defined(__unix__) || defined(__unix) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) /* snprintf, opendir */
+#elif (defined(__APPLE__) && defined(__MACH__)) || \
+     ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */
 #  define UTIL_HAS_CREATEFILELIST
 #  include <dirent.h>       /* opendir, readdir */
 #  include <errno.h>

programs/zstdcli.c

@@ -262,7 +262,7 @@ int main(int argCount, char** argv)
             if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
             if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
             if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
-            if (!strcmp(argument, "--ultra")) { ultra=1; FIO_setMaxWLog(0); continue; }
+            if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
             if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
             if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
             if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
@@ -334,8 +334,10 @@ int main(int argCount, char** argv)
                         /* destination file name */
                     case 'o': nextArgumentIsOutFileName=1; argument++; break;
 
+#ifdef UTIL_HAS_CREATEFILELIST
                         /* recursive */
                     case 'r': recursive=1; argument++; break;
+#endif
 
 #ifndef ZSTD_NOBENCH
                         /* Benchmark */

tests/datagencli.c

@@ -9,7 +9,7 @@
 
 
 /*-************************************
-*  Includes
+*  Dependencies
 **************************************/
 #include "util.h"      /* Compiler options */
 #include <stdio.h>     /* fprintf, stderr */

tests/fullbench.c

@@ -121,13 +121,12 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co
 }
 
 extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
-extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 tableRepeatFlag, const void* src, size_t srcSize);
+extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize);
 size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
 {
-    U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)];   /* MLFSELog, LLFSELog and OffFSELog are not public values */
     int nbSeq;
     (void)src; (void)srcSize; (void)dst; (void)dstSize;
-    return ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, 0, buff2, g_cSize);
+    return ZSTD_decodeSeqHeaders(g_zdc, &nbSeq, buff2, g_cSize);
 }
 
 
@@ -289,6 +288,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
             }
             iend = ip + ZSTD_blockHeaderSize + cBlockSize;   /* End of first block */
             ip += ZSTD_blockHeaderSize;                      /* skip block header */
+            ZSTD_decompressBegin(g_zdc);
             ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip);   /* skip literal segment */
             g_cSize = iend-ip;
             memcpy(buff2, ip, g_cSize);   /* copy rest of block (it starts by SeqHeader) */

tests/fuzzer.c

@@ -57,7 +57,7 @@ static U32 g_displayLevel = 2;
             if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
             { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
             if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC * 150 / 1000;
+static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
 static clock_t g_displayClock = 0;
 
 

tests/playTests.sh

@@ -205,7 +205,7 @@ $MD5SUM dirTestDict/* > tmph1
 $ZSTD -f --rm dirTestDict/* -D tmpDictC
 $ZSTD -d --rm dirTestDict/*.zst -D tmpDictC  # note : use internal checksum by default
 if [[ "$OSTYPE" == "darwin"* ]]; then
-  $ECHO "test skipped on OS-X"  # not compatible with OS-X's md5
+  $ECHO "md5sum -c not supported on OS-X : test skipped"  # not compatible with OS-X's md5
 else
   $MD5SUM -c tmph1
 fi

tests/zstreamtest.c

@@ -209,7 +209,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
 
     /* Byte-by-byte decompression test */
     DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
-    {   size_t r = 1;
+    {   /* skippable frame */
+        size_t r = 1;
         ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
         inBuff.src = compressedBuffer;
         outBuff.dst = decodedBuffer;
@@ -221,9 +222,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
             r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
             if (ZSTD_isError(r)) goto _output_error;
         }
+        /* normal frame */
         ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
         r=1;
-        while (r) {   /* normal frame */
+        while (r) {
             inBuff.size = inBuff.pos + 1;
             outBuff.size = outBuff.pos + 1;
             r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
@@ -322,6 +324,8 @@ _output_error:
 }
 
 
+/* ======   Fuzzer tests   ====== */
+
 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
 {
     const BYTE* b1 = (const BYTE*)buf1;
@@ -413,8 +417,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         FUZ_rand(&coreSeed);
         lseed = coreSeed ^ prime1;
 
-        /* states full reset (unsynchronized) */
+        /* states full reset (deliberately not synchronized) */
-        /* some issues only happen when reusing states in a specific sequence of parameters */
+        /* some issues can only happen when reusing states */
         if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); }
         if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); }
 

zstd_compression_format.md

@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.2.0 (22/07/16)
+0.2.2 (14/09/16)
 
 
 Introduction
@@ -1049,15 +1049,23 @@ by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
 
 #### Bitstream
 
-All sequences are stored in a single bitstream, read _backward_.
+FSE bitstreams are read in reverse direction than written. In zstd,
-It is therefore necessary to know the bitstream size,
+the compressor writes bits forward into a block and the decompressor
-which is deducted from compressed block size.
+must read the bitstream _backwards_.
 
-The last useful bit of the stream is followed by an end-bit-flag.
+To find the start of the bitstream it is therefore necessary to
-Highest bit of last byte is this flag.
+know the offset of the last byte of the block which can be found
-It does not belong to the useful part of the bitstream.
+by counting `Block_Size` bytes after the block header.
-Therefore, last byte has 0-7 useful bits.
+
-Note that it also means that last byte cannot be `0`.
+After writing the last bit containing information, the compressor
+writes a single `1`-bit and then fills the byte with 0-7 `0` bits of
+padding. The last byte of the compressed bitstream cannot be `0` for
+that reason.
+
+When decompressing, the last byte containing the padding is the first
+byte to read. The decompressor needs to skip 0-7 initial `0`-bits and
+the first `1`-bit it occurs. Afterwards, the useful part of the bitstream
+begins.
 
 ##### Starting states
 
@@ -1164,9 +1172,192 @@ __`Content`__ : The rest of the dictionary is its content.
 
 [compressed blocks]: #the-format-of-compressed_block
 
+Appendix A - Decoding tables for predefined codes
+-------------------------------------------------
+
+This appendix contains FSE decoding tables for the predefined literal length, match length, and offset
+codes. The tables have been constructed using the algorithm as given above in the
+"from normalized distribution to decoding tables" chapter. The tables here can be used as examples
+to crosscheck that an implementation implements the decoding table generation algorithm correctly.
+
+#### Literal Length Code:
+
+| State | Symbol | Number_Of_Bits | Base |
+| ----- | ------ | -------------- | ---- |
+|     0 |      0 |              4 |    0 |
+|     1 |      0 |              4 |   16 |
+|     2 |      1 |              5 |   32 |
+|     3 |      3 |              5 |    0 |
+|     4 |      4 |              5 |    0 |
+|     5 |      6 |              5 |    0 |
+|     6 |      7 |              5 |    0 |
+|     7 |      9 |              5 |    0 |
+|     8 |     10 |              5 |    0 |
+|     9 |     12 |              5 |    0 |
+|    10 |     14 |              6 |    0 |
+|    11 |     16 |              5 |    0 |
+|    12 |     18 |              5 |    0 |
+|    13 |     19 |              5 |    0 |
+|    14 |     21 |              5 |    0 |
+|    15 |     22 |              5 |    0 |
+|    16 |     24 |              5 |    0 |
+|    17 |     25 |              5 |   32 |
+|    18 |     26 |              5 |    0 |
+|    19 |     27 |              6 |    0 |
+|    20 |     29 |              6 |    0 |
+|    21 |     31 |              6 |    0 |
+|    22 |      0 |              4 |   32 |
+|    23 |      1 |              4 |    0 |
+|    24 |      2 |              5 |    0 |
+|    25 |      4 |              5 |   32 |
+|    26 |      5 |              5 |    0 |
+|    27 |      7 |              5 |   32 |
+|    28 |      8 |              5 |    0 |
+|    29 |     10 |              5 |   32 |
+|    30 |     11 |              5 |    0 |
+|    31 |     13 |              6 |    0 |
+|    32 |     16 |              5 |   32 |
+|    33 |     17 |              5 |    0 |
+|    34 |     19 |              5 |   32 |
+|    35 |     20 |              5 |    0 |
+|    36 |     22 |              5 |   32 |
+|    37 |     23 |              5 |    0 |
+|    38 |     25 |              4 |    0 |
+|    39 |     25 |              4 |   16 |
+|    40 |     26 |              5 |   32 |
+|    41 |     28 |              6 |    0 |
+|    42 |     30 |              6 |    0 |
+|    43 |      0 |              4 |   48 |
+|    44 |      1 |              4 |   16 |
+|    45 |      2 |              5 |   32 |
+|    46 |      3 |              5 |   32 |
+|    47 |      5 |              5 |   32 |
+|    48 |      6 |              5 |   32 |
+|    49 |      8 |              5 |   32 |
+|    50 |      9 |              5 |   32 |
+|    51 |     11 |              5 |   32 |
+|    52 |     12 |              5 |   32 |
+|    53 |     15 |              6 |    0 |
+|    54 |     17 |              5 |   32 |
+|    55 |     18 |              5 |   32 |
+|    56 |     20 |              5 |   32 |
+|    57 |     21 |              5 |   32 |
+|    58 |     23 |              5 |   32 |
+|    59 |     24 |              5 |   32 |
+|    60 |     35 |              6 |    0 |
+|    61 |     34 |              6 |    0 |
+|    62 |     33 |              6 |    0 |
+|    63 |     32 |              6 |    0 |
+
+#### Match Length Code:
+
+| State | Symbol | Number_Of_Bits | Base |
+| ----- | ------ | -------------- | ---- |
+|     0 |      0 |              6 |    0 |
+|     1 |      1 |              4 |    0 |
+|     2 |      2 |              5 |   32 |
+|     3 |      3 |              5 |    0 |
+|     4 |      5 |              5 |    0 |
+|     5 |      6 |              5 |    0 |
+|     6 |      8 |              5 |    0 |
+|     7 |     10 |              6 |    0 |
+|     8 |     13 |              6 |    0 |
+|     9 |     16 |              6 |    0 |
+|    10 |     19 |              6 |    0 |
+|    11 |     22 |              6 |    0 |
+|    12 |     25 |              6 |    0 |
+|    13 |     28 |              6 |    0 |
+|    14 |     31 |              6 |    0 |
+|    15 |     33 |              6 |    0 |
+|    16 |     35 |              6 |    0 |
+|    17 |     37 |              6 |    0 |
+|    18 |     39 |              6 |    0 |
+|    19 |     41 |              6 |    0 |
+|    20 |     43 |              6 |    0 |
+|    21 |     45 |              6 |    0 |
+|    22 |      1 |              4 |   16 |
+|    23 |      2 |              4 |    0 |
+|    24 |      3 |              5 |   32 |
+|    25 |      4 |              5 |    0 |
+|    26 |      6 |              5 |   32 |
+|    27 |      7 |              5 |    0 |
+|    28 |      9 |              6 |    0 |
+|    29 |     12 |              6 |    0 |
+|    30 |     15 |              6 |    0 |
+|    31 |     18 |              6 |    0 |
+|    32 |     21 |              6 |    0 |
+|    33 |     24 |              6 |    0 |
+|    34 |     27 |              6 |    0 |
+|    35 |     30 |              6 |    0 |
+|    36 |     32 |              6 |    0 |
+|    37 |     34 |              6 |    0 |
+|    38 |     36 |              6 |    0 |
+|    39 |     38 |              6 |    0 |
+|    40 |     40 |              6 |    0 |
+|    41 |     42 |              6 |    0 |
+|    42 |     44 |              6 |    0 |
+|    43 |      1 |              4 |   32 |
+|    44 |      1 |              4 |   48 |
+|    45 |      2 |              4 |   16 |
+|    46 |      4 |              5 |   32 |
+|    47 |      5 |              5 |   32 |
+|    48 |      7 |              5 |   32 |
+|    49 |      8 |              5 |   32 |
+|    50 |     11 |              6 |    0 |
+|    51 |     14 |              6 |    0 |
+|    52 |     17 |              6 |    0 |
+|    53 |     20 |              6 |    0 |
+|    54 |     23 |              6 |    0 |
+|    55 |     26 |              6 |    0 |
+|    56 |     29 |              6 |    0 |
+|    57 |     52 |              6 |    0 |
+|    58 |     51 |              6 |    0 |
+|    59 |     50 |              6 |    0 |
+|    60 |     49 |              6 |    0 |
+|    61 |     48 |              6 |    0 |
+|    62 |     47 |              6 |    0 |
+|    63 |     46 |              6 |    0 |
+
+#### Offset Code:
+
+| State | Symbol | Number_Of_Bits | Base |
+| ----- | ------ | -------------- | ---- |
+|     0 |      0 |              5 |    0 |
+|     1 |      6 |              4 |    0 |
+|     2 |      9 |              5 |    0 |
+|     3 |     15 |              5 |    0 |
+|     4 |     21 |              5 |    0 |
+|     5 |      3 |              5 |    0 |
+|     6 |      7 |              4 |    0 |
+|     7 |     12 |              5 |    0 |
+|     8 |     18 |              5 |    0 |
+|     9 |     23 |              5 |    0 |
+|    10 |      5 |              5 |    0 |
+|    11 |      8 |              4 |    0 |
+|    12 |     14 |              5 |    0 |
+|    13 |     20 |              5 |    0 |
+|    14 |      2 |              5 |    0 |
+|    15 |      7 |              4 |   16 |
+|    16 |     11 |              5 |    0 |
+|    17 |     17 |              5 |    0 |
+|    18 |     22 |              5 |    0 |
+|    19 |      4 |              5 |    0 |
+|    20 |      8 |              4 |   16 |
+|    21 |     13 |              5 |    0 |
+|    22 |     19 |              5 |    0 |
+|    23 |      1 |              5 |    0 |
+|    24 |      6 |              4 |   16 |
+|    25 |     10 |              5 |    0 |
+|    26 |     16 |              5 |    0 |
+|    27 |     28 |              5 |    0 |
+|    28 |     27 |              5 |    0 |
+|    29 |     26 |              5 |    0 |
+|    30 |     25 |              5 |    0 |
+|    31 |     24 |              5 |    0 |
 
 Version changes
 ---------------
+- 0.2.2 : added predefined codes, by Johannes Rudolph
 - 0.2.1 : clarify field names, by Przemyslaw Skibinski
 - 0.2.0 : numerous format adjustments for zstd v0.8
 - 0.1.2 : limit Huffman tree depth to 11 bits