lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-05 19:15:58 +03:00
Code Activity

introduced dictionary format

Browse Source
This commit is contained in:
Yann Collet
2016-01-26 03:14:20 +01:00
parent bc4c8aa4b7
commit b923f65076
9 changed files with 245 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
lib/error_private.h
View File

@@ -96,6 +96,7 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
case ZSTD_error_tableLog_tooLarge: return "tableLog requires too much memory";
case ZSTD_error_maxSymbolValue_tooLarge: return "Unsupported max possible Symbol Value : too large";
case ZSTD_error_maxSymbolValue_tooSmall: return "Specified maxSymbolValue is too small";
case ZSTD_error_dictionary_corrupted: return "Dictionary is corrupted";
case ZSTD_error_maxCode:
default: return codeError;
}

1
lib/error_public.h
View File

@@ -56,6 +56,7 @@ enum {
ZSTD_error_tableLog_tooLarge,
ZSTD_error_maxSymbolValue_tooLarge,
ZSTD_error_maxSymbolValue_tooSmall,
ZSTD_error_dictionary_corrupted,
ZSTD_error_maxCode
};

152
lib/huff0.c
View File

@@ -106,7 +106,8 @@ typedef struct nodeElt_s {
@dst : destination buffer
@CTable : huffman tree to save, using huff0 representation
@return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
{
BYTE bitsToWeight[HUF_MAX_TABLELOG + 1];
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
@@ -172,6 +173,68 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, U3
}
static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize);
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
size_t iSize;
U32 nbSymbols = 0;
U32 n;
U32 nextRankStart;
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
/* get symbol weights */
iSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* check result */
if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
/* Prepare base value per rank */
nextRankStart = 0;
for (n=1; n<=tableLog; n++) {
U32 current = nextRankStart;
nextRankStart += (rankVal[n] << (n-1));
rankVal[n] = current;
}
/* fill nbBits */
for (n=0; n<nbSymbols; n++) {
const U32 w = huffWeight[n];
CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
}
/* fill val */
{
U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
for (n=0; n<nbSymbols; n++)
nbPerRank[CTable[n].nbBits]++;
{
/* determine stating value per rank */
U16 min = 0;
for (n=HUF_MAX_TABLELOG; n>0; n--)
{
valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
}
}
for (n=0; n<=maxSymbolValue; n++)
CTable[n].val = valPerRank[CTable[n].nbBits]++; /* assign value within rank, symbol order */
}
return iSize;
}
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
{
int totalCost = 0;
@@ -384,7 +447,7 @@ size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
#define HUF_FLUSHBITS_2(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream)
size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
const BYTE* ip = (const BYTE*) src;
BYTE* const ostart = (BYTE*)dst;
@@ -429,7 +492,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size
}
static size_t HUF_compress_into4Segments(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
size_t segmentSize = (srcSize+3)/4; /* first 3 segments */
size_t errorCode;
@@ -443,28 +506,28 @@ static size_t HUF_compress_into4Segments(void* dst, size_t dstSize, const void*
if (srcSize < 12) return 0; /* no saving possible : too small input */
op += 6; /* jumpTable */
errorCode = HUF_compress_usingCTable(op, oend-op, ip, segmentSize, CTable);
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
MEM_writeLE16(ostart, (U16)errorCode);
ip += segmentSize;
op += errorCode;
errorCode = HUF_compress_usingCTable(op, oend-op, ip, segmentSize, CTable);
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
MEM_writeLE16(ostart+2, (U16)errorCode);
ip += segmentSize;
op += errorCode;
errorCode = HUF_compress_usingCTable(op, oend-op, ip, segmentSize, CTable);
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
MEM_writeLE16(ostart+4, (U16)errorCode);
ip += segmentSize;
op += errorCode;
errorCode = HUF_compress_usingCTable(op, oend-op, ip, iend-ip, CTable);
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
@@ -488,7 +551,7 @@ static size_t HUF_compress_internal (
size_t errorCode;
/* checks & inits */
if (srcSize < 2) return 0; /* Uncompressed */
if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */
if (dstSize < 1) return 0; /* not compressible within dst budget */
if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */
if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
@@ -514,9 +577,9 @@ static size_t HUF_compress_internal (
/* Compress */
if (singleStream)
errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */
errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */
else
errorCode = HUF_compress_into4Segments(op, oend - op, src, srcSize, CTable);
errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
op += errorCode;
@@ -540,55 +603,7 @@ size_t HUF_compress2 (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog)
{
#if 1
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
#else
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 count[HUF_MAX_SYMBOL_VALUE+1];
HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1];
size_t errorCode;
/* checks & inits */
if (srcSize < 1) return 0; /* Uncompressed */
if (dstSize < 1) return 0; /* not compressible within dst budget */
if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */
if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
/* Scan input and build symbol stats */
errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }
if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */
/* Build Huffman Tree */
errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
if (HUF_isError(errorCode)) return errorCode;
huffLog = (U32)errorCode;
/* Write table description header */
errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */
op += errorCode;
/* Compress */
//if (srcSize < MIN_4STREAMS) errorCode = HUF_compress_usingCTable(op, oend - op, src, srcSize, CTable); else /* single segment */
errorCode = HUF_compress_into4Segments(op, oend - op, src, srcSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
op += errorCode;
/* check compressibility */
if ((size_t)(op-ostart) >= srcSize-1)
return 0;
return op-ostart;
#endif
}
@@ -625,31 +640,24 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */
{
if (iSize >= (242)) /* RLE */
{
if (iSize >= 128) { /* special header */
if (iSize >= (242)) { /* RLE */
static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
oSize = l[iSize-242];
memset(huffWeight, 1, hwSize);
iSize = 0;
}
else /* Incompressible */
{
else { /* Incompressible */
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
for (n=0; n<oSize; n+=2)
{
for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
}
}
}
else /* header compressed with FSE (normal case) */
{
} } }
else { /* header compressed with FSE (normal case) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize)) return oSize;
@@ -712,7 +720,7 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
/* check result */
if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */
DTable[0] = (U16)tableLog; /* maybe should separate sizeof DTable, as allocated, from used size of DTable, in case of DTable re-use */
DTable[0] = (U16)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */
/* Prepare ranks */
nextRankStart = 0;

15
lib/huff0_static.h
View File

@@ -91,13 +91,11 @@ The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and regenerate 'CTable' using external methods.
*/
/* FSE_count() : find it within "fse.h" */
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress4X_into4Segments(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/*!
@@ -105,7 +103,6 @@ HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
@@ -119,6 +116,7 @@ size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* c
/* single stream variants */
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
@@ -129,6 +127,11 @@ size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* c
size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
/* Loading a CTable saved with HUF_writeCTable() */
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
#if defined (__cplusplus)
}
#endif

92
lib/zstd_compress.c
View File

@@ -125,6 +125,8 @@ struct ZSTD_CCtx_s
seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable;
U32* contentTable;
HUF_CElt* hufTable;
U32 flagHufTable;
};
@@ -185,18 +187,21 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
{
const U32 contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog)) * sizeof(U32);
const size_t neededSpace = tableSpace + (3*blockSize);
const size_t neededSpace = tableSpace + (256*sizeof(U32)) + (3*blockSize);
if (zc->workSpaceSize < neededSpace)
{
free(zc->workSpace);
zc->workSpaceSize = neededSpace;
zc->workSpace = malloc(neededSpace);
if (zc->workSpace == NULL) return ERROR(memory_allocation);
zc->workSpaceSize = neededSpace;
}
memset(zc->workSpace, 0, tableSpace );
memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
zc->hashTable = (U32*)(zc->workSpace);
zc->contentTable = zc->hashTable + ((size_t)1 << params.hashLog);
zc->seqStore.buffer = (void*) (zc->contentTable + ((size_t)1 << contentLog));
zc->seqStore.buffer = zc->contentTable + ((size_t)1 << contentLog);
zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
zc->flagHufTable = 0;
zc->seqStore.buffer = (U32*)(zc->seqStore.buffer) + 256;
}
zc->nextToUpdate = 1;
@@ -289,7 +294,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc,
big endian convention
1- CTable available (stored into workspace ?)
2- Small input
2- Small input (fast heuristic ? Full comparison ? depend on clevel ?)
1.2) Literal block content
@@ -382,21 +387,32 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t maxDstSize, const
}
size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 1; }
size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; }
static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
const size_t minGain = ZSTD_minGain(srcSize);
BYTE* const ostart = (BYTE*)dst;
size_t lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
const size_t lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
U32 singleStream = srcSize < 256;
U32 hType = IS_HUF;
size_t clitSize;
if (maxDstSize < 4) return ERROR(dstSize_tooSmall); /* not enough space for compression */
if (zc->flagHufTable && (lhSize==3))
{
hType = IS_PCH;
singleStream = 1;
clitSize = HUF_compress1X_usingCTable(ostart+lhSize, maxDstSize-lhSize, src, srcSize, zc->hufTable);
}
else
{
clitSize = singleStream ? HUF_compress1X(ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12)
: HUF_compress2 (ostart+lhSize, maxDstSize-lhSize, src, srcSize, 255, 12);
}
if ((clitSize==0) || (clitSize >= srcSize - minGain)) return ZSTD_noCompressLiterals(dst, maxDstSize, src, srcSize);
if (clitSize==1) return ZSTD_compressRleLiteralsBlock(dst, maxDstSize, src, srcSize);
@@ -405,19 +421,19 @@ static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
switch(lhSize)
{
case 3: /* 2 - 2 - 10 - 10 */
ostart[0] = (BYTE)((srcSize>>6) + (singleStream << 4));
ostart[0] = (BYTE)((srcSize>>6) + (singleStream << 4) + (hType<<6));
ostart[1] = (BYTE)((srcSize<<2) + (clitSize>>8));
ostart[2] = (BYTE)(clitSize);
break;
case 4: /* 2 - 2 - 14 - 14 */
ostart[0] = (BYTE)(srcSize>>10) + (2<<4);
ostart[0] = (BYTE)((srcSize>>10) + (2<<4) + (hType<<6));
ostart[1] = (BYTE)(srcSize>> 2);
ostart[2] = (BYTE)((srcSize<<6) + (clitSize>>8));
ostart[3] = (BYTE)(clitSize);
break;
default: /* should not be necessary, lhSize is {3,4,5} */
case 5: /* 2 - 2 - 18 - 18 */
ostart[0] = (BYTE)(srcSize>>14) + (3<<4);
ostart[0] = (BYTE)((srcSize>>14) + (3<<4) + (hType<<6));
ostart[1] = (BYTE)(srcSize>>6);
ostart[2] = (BYTE)((srcSize<<2) + (clitSize>>16));
ostart[3] = (BYTE)(clitSize>>8);
@@ -431,10 +447,11 @@ static size_t ZSTD_compressLiterals (void* dst, size_t maxDstSize,
#define LITERAL_NOENTROPY 63 /* don't even attempt to compress literals below this threshold (cheap heuristic) */
size_t ZSTD_compressSequences(void* dst, size_t maxDstSize,
const seqStore_t* seqStorePtr,
size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
void* dst, size_t maxDstSize,
size_t srcSize)
{
const seqStore_t* seqStorePtr = &(zc->seqStore);
U32 count[MaxSeq+1];
S16 norm[MaxSeq+1];
size_t mostFrequent;
@@ -463,11 +480,12 @@ size_t ZSTD_compressSequences(void* dst, size_t maxDstSize,
{
size_t cSize;
size_t litSize = seqStorePtr->lit - op_lit_start;
const size_t minLitSize = zc->flagHufTable ? 6 : LITERAL_NOENTROPY;
if (litSize <= LITERAL_NOENTROPY)
if (litSize <= minLitSize)
cSize = ZSTD_noCompressLiterals(op, maxDstSize, op_lit_start, litSize);
else
cSize = ZSTD_compressLiterals(op, maxDstSize, op_lit_start, litSize);
cSize = ZSTD_compressLiterals(zc, op, maxDstSize, op_lit_start, litSize);
if (ZSTD_isError(cSize)) return cSize;
op += cSize;
}
@@ -1905,7 +1923,7 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize);
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
{
static const ZSTD_blockCompressor blockCompressor[2][5] = {
{ ZSTD_compressBlock_fast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy,ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2 },
@@ -1921,7 +1939,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t maxDs
ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(zc->params.strategy, zc->lowLimit < zc->dictLimit);
if (srcSize < MIN_CBLOCK_SIZE+3) return 0; /* don't even attempt compression below a certain srcSize */
blockCompressor(zc, src, srcSize);
return ZSTD_compressSequences(dst, maxDstSize, &(zc->seqStore), srcSize);
return ZSTD_compressSequences(zc, dst, maxDstSize, srcSize);
}
@@ -2057,7 +2075,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t maxDstSize, const voi
}
size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t srcSize)
static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize)
{
const BYTE* const ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
@@ -2097,6 +2115,35 @@ size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* src, size_t src
}
/* Dictionary format :
Magic == ZSTD_DICT_MAGIC (4 bytes)
Huff0 CTable (256 * 4 bytes) => to be changed to read from writeCTable
Dictionary content
*/
/*! ZSTD_loadDictEntropyStats
@return : size read from dictionary */
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
/* note : magic number already checked */
const size_t hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
zc->flagHufTable = 1;
return hufHeaderSize;
}
size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
U32 magic = MEM_readLE32(dict);
U32 eSize;
if (magic != ZSTD_DICT_MAGIC)
return ZSTD_loadDictionaryContent(zc, dict, dictSize);
eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4, dictSize-4) + 4;
if (ZSTD_isError(eSize)) return eSize;
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
}
/*! ZSTD_duplicateCCtx
* Duplicate an existing context @srcCCtx into another one @dstCCtx.
* Only works during stage 0 (i.e. before first call to ZSTD_compressContinue())
@@ -2125,6 +2172,10 @@ size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
dstCCtx->dictLimit = srcCCtx->dictLimit;
dstCCtx->lowLimit = srcCCtx->lowLimit;
dstCCtx->flagHufTable = srcCCtx->flagHufTable;
if (dstCCtx->flagHufTable)
memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
return 0;
}
@@ -2164,6 +2215,11 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint)
return result;
}
/* to do
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, int compressionLevel)
{
return 0;
}*/
size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, int compressionLevel)
{

68
lib/zstd_decompress.c
View File

@@ -126,6 +126,7 @@ struct ZSTD_DCtx_s
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
U32 hufTableX4[HUF_DTABLE_SIZE(HufLog)];
const void* previousDstEnd;
const void* base;
const void* vBase;
@@ -138,7 +139,7 @@ struct ZSTD_DCtx_s
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
BYTE litBuffer[BLOCKSIZE + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
@@ -150,6 +151,7 @@ size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
dctx->base = NULL;
dctx->vBase = NULL;
dctx->dictEnd = NULL;
dctx->hufTableX4[0] = HufLog;
return 0;
}
@@ -333,6 +335,27 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
dctx->litSize = litSize;
return litCSize + lhSize;
}
case IS_PCH:
{
size_t errorCode;
size_t litSize, litCSize;
U32 lhSize = ((istart[0]) >> 4) & 3;
if (lhSize != 1) /* only case supported for now : small litSize, single stream */
return ERROR(corruption_detected);
/* 2 - 2 - 10 - 10 */
lhSize=3;
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
litCSize = ((istart[1] & 3) << 8) + istart[2];
errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
if (HUF_isError(errorCode)) return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return litCSize + lhSize;
}
case IS_RAW:
{
size_t litSize;
@@ -386,12 +409,12 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
memset(dctx->litBuffer, istart[lhSize], litSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litBufSize = BLOCKSIZE+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return lhSize+1;
}
default: /* IS_PCH */
return ERROR(corruption_detected); /* not yet nominal case */
default:
return ERROR(corruption_detected); /* impossible */
}
}
@@ -794,7 +817,8 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
ZSTD_resetDCtx(dctx);
if (dict)
{
ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
size_t errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted);
dctx->dictEnd = dctx->previousDstEnd;
dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
dctx->base = dst;
@@ -979,10 +1003,42 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
}
void ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
dctx->dictEnd = dctx->previousDstEnd;
dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
dctx->base = dict;
dctx->previousDstEnd = (const char*)dict + dictSize;
}
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
size_t hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
return hSize;
}
size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
size_t eSize;
U32 magic = MEM_readLE32(dict);
if (magic != ZSTD_DICT_MAGIC) {
/* pure content mode */
ZSTD_refDictContent(dctx, dict, dictSize);
return 0;
}
/* load entropy tables */
dict = (const char*)dict + 4;
dictSize -= 4;
eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
/* reference dictionary content */
dict = (const char*)dict + eSize;
dictSize -= eSize;
ZSTD_refDictContent(dctx, dict, dictSize);
return 0;
}

5
lib/zstd_internal.h
View File

@@ -55,7 +55,8 @@ extern "C" {
/* *************************************
* Common constants
***************************************/
#define ZSTD_MAGICNUMBER 0xFD2FB524 /* v0.4 */
#define ZSTD_MAGICNUMBER 0xFD2FB525 /* v0.5 */
#define ZSTD_DICT_MAGIC 0xEC30A435
#define KB *(1 <<10)
#define MB *(1 <<20)
@@ -93,6 +94,8 @@ static const size_t ZSTD_frameHeaderSize_min = 5;
#define OffFSELog 9
#define MaxSeq MAX(MaxLL, MaxML)
#define HufLog 12
#define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + MIN_SEQUENCES_SIZE)

5
lib/zstd_static.h
View File

@@ -121,6 +121,9 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
* Streaming functions (direct mode)
****************************************/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict,size_t dictSize, int compressionLevel);
//ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters params);
ZSTDLIB_API size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* dict, size_t dictSize);
@@ -163,7 +166,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz
ZSTDLIB_API size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
ZSTDLIB_API void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);

14
programs/bench.c
View File

@@ -310,11 +310,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (ZSTD_isError(rSize)) EXM_THROW(2, "ZSTD_compressEnd() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].cSize += rSize;
}
/*blockTable[blockNb].cSize = ZSTD_compress_usingDict(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
dictBuffer, dictBufferSize,
cLevel);*/
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
@@ -334,14 +329,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++)
{
for (blockNb=0; blockNb<nbBlocks; blockNb++)
for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++) {
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
blockTable[blockNb].resSize = ZSTD_decompress_usingDict(dctx,
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
dictBuffer, dictBufferSize);
}
if (ZSTD_isError(blockTable[blockNb].resSize))
EXM_THROW(3, "ZSTD_decompress_usingDict() failed : %s", ZSTD_getErrorName(blockTable[blockNb].resSize));
} }
milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;