introduced dictionary format

lib/error_private.h

@@ -96,6 +96,7 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
     case ZSTD_error_tableLog_tooLarge: return "tableLog requires too much memory";
     case ZSTD_error_maxSymbolValue_tooLarge: return "Unsupported max possible Symbol Value : too large";
     case ZSTD_error_maxSymbolValue_tooSmall: return "Specified maxSymbolValue is too small";
+    case ZSTD_error_dictionary_corrupted: return "Dictionary is corrupted";
     case ZSTD_error_maxCode:
     default: return codeError;
     }

lib/error_public.h

@@ -56,6 +56,7 @@ enum {
   ZSTD_error_tableLog_tooLarge,
   ZSTD_error_maxSymbolValue_tooLarge,
   ZSTD_error_maxSymbolValue_tooSmall,
+  ZSTD_error_dictionary_corrupted,
   ZSTD_error_maxCode
 };
 

lib/huff0.c

@@ -106,7 +106,8 @@ typedef struct nodeElt_s {
     @dst : destination buffer
     @CTable : huffman tree to save, using huff0 representation
     @return : size of saved CTable */
-size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
+size_t HUF_writeCTable (void* dst, size_t maxDstSize,
+                        const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
 {
     BYTE bitsToWeight[HUF_MAX_TABLELOG + 1];
     BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
@@ -172,6 +173,68 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, U3
 }
 
 
+static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
+                            U32* nbSymbolsPtr, U32* tableLogPtr,
+                            const void* src, size_t srcSize);
+
+
+size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
+{
+    BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
+    U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1];   /* large enough for values from 0 to 16 */
+    U32 tableLog = 0;
+    size_t iSize;
+    U32 nbSymbols = 0;
+    U32 n;
+    U32 nextRankStart;
+    //memset(huffWeight, 0, sizeof(huffWeight));   /* is not necessary, even though some analyzer complain ... */
+
+    /* get symbol weights */
+    iSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
+    if (HUF_isError(iSize)) return iSize;
+
+    /* check result */
+    if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
+    if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
+
+    /* Prepare base value per rank */
+    nextRankStart = 0;
+    for (n=1; n<=tableLog; n++) {
+        U32 current = nextRankStart;
+        nextRankStart += (rankVal[n] << (n-1));
+        rankVal[n] = current;
+    }
+
+    /* fill nbBits */
+    for (n=0; n<nbSymbols; n++) {
+        const U32 w = huffWeight[n];
+        CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
+    }
+
+    /* fill val */
+    {
+        U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
+        U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
+        for (n=0; n<nbSymbols; n++)
+            nbPerRank[CTable[n].nbBits]++;
+        {
+            /* determine stating value per rank */
+            U16 min = 0;
+            for (n=HUF_MAX_TABLELOG; n>0; n--)
+            {
+                valPerRank[n] = min;      /* get starting value within each rank */
+                min += nbPerRank[n];
+                min >>= 1;
+            }
+        }
+        for (n=0; n<=maxSymbolValue; n++)
+            CTable[n].val = valPerRank[CTable[n].nbBits]++;   /* assign value within rank, symbol order */
+    }
+
+    return iSize;
+}
+
+
 static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
 {
     int totalCost = 0;
@@ -384,7 +447,7 @@ size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
 #define HUF_FLUSHBITS_2(stream) \
     if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream)
 
-size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
 {
     const BYTE* ip = (const BYTE*) src;
     BYTE* const ostart = (BYTE*)dst;
@@ -429,7 +492,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size
 }
 
 
-static size_t HUF_compress_into4Segments(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
 {
     size_t segmentSize = (srcSize+3)/4;   /* first 3 segments */
     size_t errorCode;
@@ -443,28 +506,28 @@ static size_t HUF_compress_into4Segments(void* dst, size_t dstSize, const void*
     if (srcSize < 12) return 0;   /* no saving possible : too small input */
     op += 6;   /* jumpTable */
 
-    errorCode = HUF_compress_usingCTable(op, oend-op, ip, segmentSize, CTable);
+    errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
     if (HUF_isError(errorCode)) return errorCode;
     if (errorCode==0) return 0;
     MEM_writeLE16(ostart, (U16)errorCode);
 
     ip += segmentSize;
     op += errorCode;
-    errorCode = HUF_compress_usingCTable(op, oend-op, ip, segmentSize, CTable);
+    errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
     if (HUF_isError(errorCode)) return errorCode;
     if (errorCode==0) return 0;
     MEM_writeLE16(ostart+2, (U16)errorCode);
 
     ip += segmentSize;
     op += errorCode;
-    errorCode = HUF_compress_usingCTable(op, oend-op, ip, segmentSize, CTable);
+    errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
     if (HUF_isError(errorCode)) return errorCode;
     if (errorCode==0) return 0;
     MEM_writeLE16(ostart+4, (U16)errorCode);
 
     ip += segmentSize;
     op += errorCode;
-    errorCode = HUF_compress_usingCTable(op, oend-op, ip, iend-ip, CTable);
+    errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
     if (HUF_isError(errorCode)) return errorCode;
     if (errorCode==0) return 0;
 
@@ -488,7 +551,7 @@ static size_t HUF_compress_internal (
     size_t errorCode;
 
     /* checks & inits */
-    if (srcSize < 2) return 0;  /* Uncompressed */
+    if (srcSize < 1) return 0;  /* Uncompressed - note : 1 means rle, so first byte must be correct */
     if (dstSize < 1) return 0;  /* not compressible within dst budget */
     if (srcSize > 128 * 1024) return ERROR(srcSize_wrong);   /* current block size limit */
     if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
@@ -514,9 +577,9 @@ static size_t HUF_compress_internal (
 
     /* Compress */
     if (singleStream)
-        errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable);   /* single segment */
+        errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable);   /* single segment */
     else
-        errorCode = HUF_compress_into4Segments(op, oend - op, src, srcSize, CTable);
+        errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
     if (HUF_isError(errorCode)) return errorCode;
     if (errorCode==0) return 0;
     op += errorCode;
@@ -540,55 +603,7 @@ size_t HUF_compress2 (void* dst, size_t dstSize,
                 const void* src, size_t srcSize,
                 unsigned maxSymbolValue, unsigned huffLog)
 {
-#if 1
     return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
-#else
-    BYTE* const ostart = (BYTE*)dst;
-    BYTE* op = ostart;
-    BYTE* const oend = ostart + dstSize;
-
-    U32 count[HUF_MAX_SYMBOL_VALUE+1];
-    HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1];
-    size_t errorCode;
-
-    /* checks & inits */
-    if (srcSize < 1) return 0;  /* Uncompressed */
-    if (dstSize < 1) return 0;  /* not compressible within dst budget */
-    if (srcSize > 128 * 1024) return ERROR(srcSize_wrong);   /* current block size limit */
-    if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
-    if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
-    if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
-
-    /* Scan input and build symbol stats */
-    errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
-    if (HUF_isError(errorCode)) return errorCode;
-    if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }
-    if (errorCode <= (srcSize >> 7)+1) return 0;   /* Heuristic : not compressible enough */
-
-    /* Build Huffman Tree */
-    errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
-    if (HUF_isError(errorCode)) return errorCode;
-    huffLog = (U32)errorCode;
-
-    /* Write table description header */
-    errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
-    if (HUF_isError(errorCode)) return errorCode;
-    if (errorCode + 12 >= srcSize) return 0;   /* not useful to try compression */
-    op += errorCode;
-
-    /* Compress */
-    //if (srcSize < MIN_4STREAMS) errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable); else  /* single segment */
-    errorCode = HUF_compress_into4Segments(op, oend - op, src, srcSize, CTable);
-    if (HUF_isError(errorCode)) return errorCode;
-    if (errorCode==0) return 0;
-    op += errorCode;
-
-    /* check compressibility */
-    if ((size_t)(op-ostart) >= srcSize-1)
-        return 0;
-
-    return op-ostart;
-#endif
 }
 
 
@@ -625,31 +640,24 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
 
     //memset(huffWeight, 0, hwSize);   /* is not necessary, even though some analyzer complain ... */
 
-    if (iSize >= 128)  /* special header */
+    if (iSize >= 128)  { /* special header */
-    {
+        if (iSize >= (242)) {  /* RLE */
-        if (iSize >= (242))   /* RLE */
-        {
             static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
             oSize = l[iSize-242];
             memset(huffWeight, 1, hwSize);
             iSize = 0;
         }
-        else   /* Incompressible */
+        else {   /* Incompressible */
-        {
             oSize = iSize - 127;
             iSize = ((oSize+1)/2);
             if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
             if (oSize >= hwSize) return ERROR(corruption_detected);
             ip += 1;
-            for (n=0; n<oSize; n+=2)
+            for (n=0; n<oSize; n+=2) {
-            {
                 huffWeight[n]   = ip[n/2] >> 4;
                 huffWeight[n+1] = ip[n/2] & 15;
-            }
+    }   }   }
-        }
+    else  {   /* header compressed with FSE (normal case) */
-    }
-    else  /* header compressed with FSE (normal case) */
-    {
         if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
         oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize);   /* max (hwSize-1) values decoded, as last one is implied */
         if (FSE_isError(oSize)) return oSize;
@@ -712,7 +720,7 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
 
     /* check result */
     if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge);   /* DTable is too small */
-    DTable[0] = (U16)tableLog;   /* maybe should separate sizeof DTable, as allocated, from used size of DTable, in case of DTable re-use */
+    DTable[0] = (U16)tableLog;   /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */
 
     /* Prepare ranks */
     nextRankStart = 0;

lib/huff0_static.h

@@ -91,13 +91,11 @@ The following API allows targeting specific sub-functions for advanced tasks.
 For example, it's possible to compress several blocks using the same 'CTable',
 or to save and regenerate 'CTable' using external methods.
 */
-
 /* FSE_count() : find it within "fse.h" */
-
 typedef struct HUF_CElt_s HUF_CElt;   /* incomplete type */
-size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
+size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
-size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, unsigned maxSymbolValue, unsigned huffLog);
+size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
-size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
+size_t HUF_compress4X_into4Segments(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
 
 
 /*!
@@ -105,7 +103,6 @@ HUF_decompress() does the following:
 1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
 2. build Huffman table from save, using HUF_readDTableXn()
 3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
-
 */
 size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
 size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
@@ -119,6 +116,7 @@ size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* c
 /* single stream variants */
 
 size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
 
 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
 size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
@@ -129,6 +127,11 @@ size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* c
 size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
 
 
+/* Loading a CTable saved with HUF_writeCTable() */
+
+size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
+
+
 #if defined (__cplusplus)
 }
 #endif

lib/zstd_compress.c

@@ -125,6 +125,8 @@ struct ZSTD_CCtx_s
     seqStore_t seqStore;    /* sequences storage ptrs */
     U32* hashTable;
     U32* contentTable;
+    HUF_CElt* hufTable;
+    U32 flagHufTable;
 };
 
 
@@ -185,18 +187,21 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
     {
         const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
         const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog)) * sizeof(U32);
-        const size_t neededSpace = tableSpace + (3*blockSize);
+        const size_t neededSpace = tableSpace + (256*sizeof(U32)) + (3*blockSize);
         if (zc->workSpaceSize < neededSpace)
         {
             free(zc->workSpace);
-            zc->workSpaceSize = neededSpace;
             zc->workSpace = malloc(neededSpace);
             if (zc->workSpace == NULL) return ERROR(memory_allocation);
+            zc->workSpaceSize = neededSpace;
         }
-        memset(zc->workSpace, 0, tableSpace );
+        memset(zc->workSpace, 0, tableSpace );   /* reset only tables */
         zc->hashTable = (U32*)(zc->workSpace);
         zc->contentTable = zc->hashTable + ((size_t)1 << params.hashLog);
-        zc->seqStore.buffer = (void*) (zc->contentTable + ((size_t)1 << contentLog));
+        zc->seqStore.buffer = zc->contentTable + ((size_t)1 << contentLog);
+        zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
+        zc->flagHufTable = 0;
+        zc->seqStore.buffer = (U32*)(zc->seqStore.buffer) + 256;
     }
 
     zc->nextToUpdate = 1;
@@ -289,7 +294,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc,
             big endian convention
 
         1- CTable available (stored into workspace ?)
-        2- Small input
+        2- Small input (fast heuristic ? Full comparison ? depend on clevel ?)
 
 
    1.2) Literal block content
@@ -382,21 +387,32 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const
 }
 
 
-size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 1; }
+size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
 
-static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
+static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
+                                     void* dst, size_t maxDstSize,
                                const void* src, size_t srcSize)
 {
     const size_t minGain = ZSTD_minGain(srcSize);
     BYTE* const ostart = (BYTE*)dst;
-    size_t lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
+    const size_t lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
     U32 singleStream = srcSize < 256;
+    U32 hType = IS_HUF;
     size_t clitSize;
 
     if (maxDstSize < 4) return ERROR(dstSize_tooSmall);   /* not enough space for compression */
 
-    clitSize = singleStream ? HUF_compress1X(ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12)
+    if (zc->flagHufTable && (lhSize==3))
-                            : HUF_compress2 (ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12);
+    {
+        hType = IS_PCH;
+        singleStream = 1;
+        clitSize = HUF_compress1X_usingCTable(ostart+lhSize, maxDstSize-lhSize, src, srcSize, zc->hufTable);
+    }
+    else
+    {
+        clitSize = singleStream ? HUF_compress1X(ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12)
+                                : HUF_compress2 (ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12);
+    }
 
     if ((clitSize==0) || (clitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, maxDstSize, src, srcSize);
     if (clitSize==1) return ZSTD_compressRleLiteralsBlock(dst, maxDstSize, src, srcSize);
@@ -405,19 +421,19 @@ static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
     switch(lhSize)
     {
     case 3: /* 2 - 2 - 10 - 10 */
-        ostart[0] = (BYTE)((srcSize>>6) + (singleStream << 4));
+        ostart[0] = (BYTE)((srcSize>>6) + (singleStream << 4) + (hType<<6));
         ostart[1] = (BYTE)((srcSize<<2) + (clitSize>>8));
         ostart[2] = (BYTE)(clitSize);
         break;
     case 4: /* 2 - 2 - 14 - 14 */
-        ostart[0] = (BYTE)(srcSize>>10) + (2<<4);
+        ostart[0] = (BYTE)((srcSize>>10) + (2<<4) +  (hType<<6));
         ostart[1] = (BYTE)(srcSize>> 2);
         ostart[2] = (BYTE)((srcSize<<6) + (clitSize>>8));
         ostart[3] = (BYTE)(clitSize);
         break;
     default:   /* should not be necessary, lhSize is {3,4,5} */
     case 5: /* 2 - 2 - 18 - 18 */
-        ostart[0] = (BYTE)(srcSize>>14) + (3<<4);
+        ostart[0] = (BYTE)((srcSize>>14) + (3<<4) +  (hType<<6));
         ostart[1] = (BYTE)(srcSize>>6);
         ostart[2] = (BYTE)((srcSize<<2) + (clitSize>>16));
         ostart[3] = (BYTE)(clitSize>>8);
@@ -431,10 +447,11 @@ static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
 
 #define LITERAL_NOENTROPY 63   /* don't even attempt to compress literals below this threshold (cheap heuristic) */
 
-size_t ZSTD_compressSequences(void* dst, size_t maxDstSize,
+size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
-                        const seqStore_t* seqStorePtr,
+                              void* dst, size_t maxDstSize,
                               size_t srcSize)
 {
+    const seqStore_t* seqStorePtr = &(zc->seqStore);
     U32 count[MaxSeq+1];
     S16 norm[MaxSeq+1];
     size_t mostFrequent;
@@ -463,11 +480,12 @@ size_t ZSTD_compressSequences(void* dst, size_t maxDstSize,
     {
         size_t cSize;
         size_t litSize = seqStorePtr->lit - op_lit_start;
+        const size_t minLitSize = zc->flagHufTable ? 6 : LITERAL_NOENTROPY;
 
-        if (litSize <= LITERAL_NOENTROPY)
+        if (litSize <= minLitSize)
             cSize = ZSTD_noCompressLiterals(op, maxDstSize, op_lit_start, litSize);
         else
-            cSize = ZSTD_compressLiterals(op, maxDstSize, op_lit_start, litSize);
+            cSize = ZSTD_compressLiterals(zc, op, maxDstSize, op_lit_start, litSize);
         if (ZSTD_isError(cSize)) return cSize;
         op += cSize;
     }
@@ -1905,7 +1923,7 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
 
 typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize);
 
-ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
+static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
 {
     static const ZSTD_blockCompressor blockCompressor[2][5] = {
         { ZSTD_compressBlock_fast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy,ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2 },
@@ -1921,7 +1939,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDs
     ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit);
     if (srcSize < MIN_CBLOCK_SIZE+3) return 0;   /* don't even attempt compression below a certain srcSize */
     blockCompressor(zc, src, srcSize);
-    return ZSTD_compressSequences(dst, maxDstSize, &(zc->seqStore), srcSize);
+    return ZSTD_compressSequences(zc, dst, maxDstSize, srcSize);
 }
 
 
@@ -2057,7 +2075,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const voi
 }
 
 
-size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize)
+static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize)
 {
     const BYTE* const ip = (const BYTE*) src;
     const BYTE* const iend = ip + srcSize;
@@ -2097,6 +2115,35 @@ size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t src
 }
 
 
+/* Dictionary format :
+     Magic == ZSTD_DICT_MAGIC (4 bytes)
+     Huff0 CTable (256 * 4 bytes)  => to be changed to read from writeCTable
+     Dictionary content
+*/
+/*! ZSTD_loadDictEntropyStats
+    @return : size read from dictionary */
+static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
+{
+    /* note : magic number already checked */
+    const size_t hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
+    if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
+    zc->flagHufTable = 1;
+    return hufHeaderSize;
+}
+
+size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
+{
+    U32 magic = MEM_readLE32(dict);
+    U32 eSize;
+    if (magic != ZSTD_DICT_MAGIC)
+        return ZSTD_loadDictionaryContent(zc, dict, dictSize);
+
+    eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4, dictSize-4) + 4;
+    if (ZSTD_isError(eSize)) return eSize;
+    return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
+}
+
+
 /*! ZSTD_duplicateCCtx
 *   Duplicate an existing context @srcCCtx into another one @dstCCtx.
 *   Only works during stage 0 (i.e. before first call to ZSTD_compressContinue())
@@ -2125,6 +2172,10 @@ size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
     dstCCtx->dictLimit   = srcCCtx->dictLimit;
     dstCCtx->lowLimit    = srcCCtx->lowLimit;
 
+    dstCCtx->flagHufTable = srcCCtx->flagHufTable;
+    if (dstCCtx->flagHufTable)
+        memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
+
     return 0;
 }
 
@@ -2164,6 +2215,11 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint)
     return result;
 }
 
+/* to do
+size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, int compressionLevel)
+{
+    return 0;
+}*/
 
 size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, int compressionLevel)
 {

lib/zstd_decompress.c

@@ -126,6 +126,7 @@ struct ZSTD_DCtx_s
     U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
     U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
     U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+    U32 hufTableX4[HUF_DTABLE_SIZE(HufLog)];
     const void* previousDstEnd;
     const void* base;
     const void* vBase;
@@ -138,7 +139,7 @@ struct ZSTD_DCtx_s
     const BYTE* litPtr;
     size_t litBufSize;
     size_t litSize;
-    BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
+    BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
     BYTE headerBuffer[ZSTD_frameHeaderSize_max];
 };  /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
 
@@ -150,6 +151,7 @@ size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
     dctx->base = NULL;
     dctx->vBase = NULL;
     dctx->dictEnd = NULL;
+    dctx->hufTableX4[0] = HufLog;
     return 0;
 }
 
@@ -333,6 +335,27 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
             dctx->litSize = litSize;
             return litCSize + lhSize;
         }
+    case IS_PCH:
+        {
+            size_t errorCode;
+            size_t litSize, litCSize;
+            U32 lhSize = ((istart[0]) >> 4) & 3;
+            if (lhSize != 1)  /* only case supported for now : small litSize, single stream */
+                return ERROR(corruption_detected);
+
+            /* 2 - 2 - 10 - 10 */
+            lhSize=3;
+            litSize  = ((istart[0] & 15) << 6) + (istart[1] >> 2);
+            litCSize = ((istart[1] &  3) << 8) + istart[2];
+
+            errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
+            if (HUF_isError(errorCode)) return ERROR(corruption_detected);
+
+            dctx->litPtr = dctx->litBuffer;
+            dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
+            dctx->litSize = litSize;
+            return litCSize + lhSize;
+        }
     case IS_RAW:
         {
             size_t litSize;
@@ -386,12 +409,12 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
             if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
             memset(dctx->litBuffer, istart[lhSize], litSize);
             dctx->litPtr = dctx->litBuffer;
-            dctx->litBufSize = BLOCKSIZE+8;
+            dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
             dctx->litSize = litSize;
             return lhSize+1;
         }
-    default: /* IS_PCH */
+    default:
-        return ERROR(corruption_detected);   /* not yet nominal case */
+        return ERROR(corruption_detected);   /* impossible */
     }
 }
 
@@ -794,7 +817,8 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
     ZSTD_resetDCtx(dctx);
     if (dict)
     {
-        ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
+        size_t errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
+        if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted);
         dctx->dictEnd = dctx->previousDstEnd;
         dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
         dctx->base = dst;
@@ -979,10 +1003,42 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
 }
 
 
-void ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
 {
     dctx->dictEnd = dctx->previousDstEnd;
     dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
     dctx->base = dict;
     dctx->previousDstEnd = (const char*)dict + dictSize;
 }
+
+
+static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+    size_t hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
+    if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
+    return hSize;
+}
+
+size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
+{
+    size_t eSize;
+    U32 magic = MEM_readLE32(dict);
+    if (magic != ZSTD_DICT_MAGIC) {
+        /* pure content mode */
+        ZSTD_refDictContent(dctx, dict, dictSize);
+        return 0;
+    }
+    /* load entropy tables */
+    dict = (const char*)dict + 4;
+    dictSize -= 4;
+    eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
+    if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
+
+    /* reference dictionary content */
+    dict = (const char*)dict + eSize;
+    dictSize -= eSize;
+    ZSTD_refDictContent(dctx, dict, dictSize);
+
+    return 0;
+}
+

lib/zstd_internal.h

@@ -55,7 +55,8 @@ extern "C" {
 /* *************************************
 *  Common constants
 ***************************************/
-#define ZSTD_MAGICNUMBER 0xFD2FB524   /* v0.4 */
+#define ZSTD_MAGICNUMBER 0xFD2FB525   /* v0.5 */
+#define ZSTD_DICT_MAGIC  0xEC30A435
 
 #define KB *(1 <<10)
 #define MB *(1 <<20)
@@ -93,6 +94,8 @@ static const size_t ZSTD_frameHeaderSize_min = 5;
 #define OffFSELog   9
 #define MaxSeq MAX(MaxLL, MaxML)
 
+#define HufLog 12
+
 #define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
 #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + MIN_SEQUENCES_SIZE)
 

lib/zstd_static.h

@@ -121,6 +121,9 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
 *  Streaming functions (direct mode)
 ****************************************/
 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, int compressionLevel);
+//ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params);
+
 ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters params);
 
 ZSTDLIB_API size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* dict, size_t dictSize);
@@ -163,7 +166,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz
 
 ZSTDLIB_API size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
 ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
-ZSTDLIB_API void   ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
+ZSTDLIB_API size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
 
 ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
 ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);

programs/bench.c

@@ -310,11 +310,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                     if (ZSTD_isError(rSize)) EXM_THROW(2, "ZSTD_compressEnd() failed : %s", ZSTD_getErrorName(rSize));
                     blockTable[blockNb].cSize += rSize;
                 }
-                    /*blockTable[blockNb].cSize = ZSTD_compress_usingDict(ctx,
-                                                              blockTable[blockNb].cPtr,  blockTable[blockNb].cRoom,
-                                                              blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
-                                                              dictBuffer, dictBufferSize,
-                                                              cLevel);*/
                 nbLoops++;
             }
             milliTime = BMK_GetMilliSpan(milliTime);
@@ -334,14 +329,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             milliTime = BMK_GetMilliStart();
             while (BMK_GetMilliStart() == milliTime);
             milliTime = BMK_GetMilliStart();
-            for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++)
+            for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++) {
-            {
+                for (blockNb=0; blockNb<nbBlocks; blockNb++) {
-                for (blockNb=0; blockNb<nbBlocks; blockNb++)
                     blockTable[blockNb].resSize = ZSTD_decompress_usingDict(dctx,
-                                                                blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
+                        blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
-                                                                blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
+                        blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
-                                                                dictBuffer, dictBufferSize);
+                        dictBuffer, dictBufferSize);
-            }
+                    if (ZSTD_isError(blockTable[blockNb].resSize))
+                        EXM_THROW(3, "ZSTD_decompress_usingDict() failed : %s", ZSTD_getErrorName(blockTable[blockNb].resSize));
+            }   }
             milliTime = BMK_GetMilliSpan(milliTime);
 
             if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;