lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-29 11:21:22 +03:00
Code Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev

Browse Source
This commit is contained in:
inikep
2016-09-15 20:41:40 +02:00
parent dd8905b351 55f276949c
commit 3e70cf8036
15 changed files with 471 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
NEWS
View File

@ -4,8 +4,10 @@ added : NetBSD install target (#338)
Improved : variable compression speed improvements on batches of small files.
Fixed : CLI -d output to stdout by default when input is stdin (#322)
Fixed : CLI correctly detects console on Mac OS-X
Fixed : Legacy decoders use unified error codes (#341), reported by benrg
Fixed : CLI supports recursive mode `-r` on Mac OS-X
Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
Fixed : zstd-pgo, reported by octoploid (#329)
v1.0.0

70
lib/compress/zstd_compress.c
View File

@ -113,6 +113,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
{
if (cctx==NULL) return 0; /* support sizeof on NULL */
return sizeof(*cctx) + cctx->workSpaceSize;
}
@ -168,7 +169,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
U32 const srcLog = ZSTD_highbit32((U32)(rSize)-1) + 1;
U32 const srcLog = MAX(4, ZSTD_highbit32((U32)(rSize)-1) + 1);
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
@ -238,9 +239,9 @@ typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_comp
note : 'params' must be validated */
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
ZSTD_parameters params, U64 frameContentSize,
ZSTD_compResetPolicy_e crp)
ZSTD_compResetPolicy_e const crp)
{
if (crp == ZSTDcrp_continue) /* still some issues */
if (crp == ZSTDcrp_continue)
if (ZSTD_equivalentParams(params, zc->params))
return ZSTD_continueCCtx(zc, params, frameContentSize);
@ -2680,6 +2681,12 @@ struct ZSTD_CDict_s {
ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
{
if (cdict==NULL) return 0; /* support sizeof on NULL */
return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize;
}
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
@ -2731,17 +2738,23 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
}
}
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
{
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext))
else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
return 0;
}
/*! ZSTD_compress_usingCDict() :
* Compression using a digested Dictionary.
* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
* Note that compression level is decided during dictionary creation */
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict)
{
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext))
else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, srcSize));
CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));
if (cdict->refContext->params.fParams.contentSizeFlag==1) {
cctx->params.fParams.contentSizeFlag = 1;
@ -2760,7 +2773,8 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
struct ZSTD_CStream_s {
ZSTD_CCtx* zc;
ZSTD_CCtx* cctx;
ZSTD_CDict* cdict;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
@ -2793,8 +2807,8 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
if (zcs==NULL) return NULL;
memset(zcs, 0, sizeof(ZSTD_CStream));
memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
zcs->zc = ZSTD_createCCtx_advanced(customMem);
if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; }
zcs->cctx = ZSTD_createCCtx_advanced(customMem);
if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; }
return zcs;
}
@ -2802,7 +2816,8 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
{
if (zcs==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = zcs->customMem;
ZSTD_freeCCtx(zcs->zc);
ZSTD_freeCCtx(zcs->cctx);
ZSTD_freeCDict(zcs->cdict);
ZSTD_free(zcs->inBuff, cMem);
ZSTD_free(zcs->outBuff, cMem);
ZSTD_free(zcs, cMem);
@ -2816,6 +2831,19 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{
CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize));
zcs->inToCompress = 0;
zcs->inBuffPos = 0;
zcs->inBuffTarget = zcs->blockSize;
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load;
zcs->frameEnded = 0;
return 0; /* ready to go */
}
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
@ -2837,16 +2865,13 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
if (zcs->outBuff == NULL) return ERROR(memory_allocation);
}
CHECK_F(ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize));
ZSTD_freeCDict(zcs->cdict);
zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
if (zcs->cdict == NULL) return ERROR(memory_allocation);
zcs->inToCompress = 0;
zcs->inBuffPos = 0;
zcs->inBuffTarget = zcs->blockSize;
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load;
zcs->checksum = params.fParams.checksumFlag > 0;
zcs->frameEnded = 0;
return 0; /* ready to go */
return ZSTD_resetCStream(zcs, pledgedSrcSize);
}
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
@ -2862,7 +2887,8 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
{
return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize;
if (zcs==NULL) return 0; /* support sizeof on NULL */
return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize;
}
/*====== Compression ======*/
@ -2913,8 +2939,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
else
cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = (flush == zsf_end) ?
ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize;
if (flush == zsf_end) zcs->frameEnded = 1;
/* prepare next block */
@ -3008,7 +3034,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
/* create epilogue */
zcs->stage = zcss_final;
zcs->outBuffContentSize = !notEnded ? 0 :
ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */
ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */
}
/* flush epilogue */

179
lib/decompress/zstd_decompress.c
View File

@ -80,8 +80,12 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
struct ZSTD_DCtx_s
{
const FSE_DTable* LLTptr;
const FSE_DTable* MLTptr;
const FSE_DTable* OFTptr;
const HUF_DTable* HUFptr;
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
const void* previousDstEnd;
@ -107,7 +111,7 @@ struct ZSTD_DCtx_s
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); }
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; return sizeof(ZSTD_DCtx); } /* support sizeof on NULL */
size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
@ -119,11 +123,15 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
dctx->base = NULL;
dctx->vBase = NULL;
dctx->dictEnd = NULL;
dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);
dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
dctx->litEntropy = dctx->fseEntropy = 0;
dctx->dictID = 0;
MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue));
memcpy(dctx->rep, repStartValue, sizeof(repStartValue));
memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
dctx->LLTptr = dctx->LLTable;
dctx->MLTptr = dctx->MLTable;
dctx->OFTptr = dctx->OFTable;
dctx->HUFptr = dctx->hufTable;
return 0;
}
@ -159,6 +167,25 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
}
static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
{
ZSTD_decompressBegin(dstDCtx); /* init */
dstDCtx->dictEnd = srcDCtx->dictEnd;
dstDCtx->vBase = srcDCtx->vBase;
dstDCtx->base = srcDCtx->base;
dstDCtx->previousDstEnd = srcDCtx->previousDstEnd;
dstDCtx->dictID = srcDCtx->dictID;
dstDCtx->litEntropy = srcDCtx->litEntropy;
dstDCtx->fseEntropy = srcDCtx->fseEntropy;
dstDCtx->LLTptr = srcDCtx->LLTable;
dstDCtx->MLTptr = srcDCtx->MLTable;
dstDCtx->OFTptr = srcDCtx->OFTable;
dstDCtx->HUFptr = srcDCtx->hufTable;
dstDCtx->rep[0] = srcDCtx->rep[0];
dstDCtx->rep[1] = srcDCtx->rep[1];
dstDCtx->rep[2] = srcDCtx->rep[2];
}
/*-*************************************************************
* Decompression section
@ -350,34 +377,31 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
{
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
/* 2 - 2 - 10 - 10 */
{ singleStream = !lhlCode;
singleStream = !lhlCode;
lhSize = 3;
litSize = (lhc >> 4) & 0x3FF;
litCSize = (lhc >> 14) & 0x3FF;
break;
}
case 2:
/* 2 - 2 - 14 - 14 */
{ lhSize = 4;
lhSize = 4;
litSize = (lhc >> 4) & 0x3FFF;
litCSize = lhc >> 18;
break;
}
case 3:
/* 2 - 2 - 18 - 18 */
{ lhSize = 5;
lhSize = 5;
litSize = (lhc >> 4) & 0x3FFFF;
litCSize = (lhc >> 22) + (istart[4] << 10);
break;
}
}
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
if (HUF_isError((litEncType==set_repeat) ?
( singleStream ?
HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) :
HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) :
HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) :
HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) :
( singleStream ?
HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
@ -387,6 +411,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable;
return litCSize + lhSize;
}
@ -461,7 +486,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
@return : nb bytes read from src,
or an error code if it fails, testable with ZSTD_isError()
*/
FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog,
static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog,
const void* src, size_t srcSize,
const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
{
@ -491,8 +516,7 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e
}
size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 flagRepeatTable,
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
const void* src, size_t srcSize)
{
const BYTE* const istart = (const BYTE* const)src;
@ -522,16 +546,19 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
ip++;
/* Build DTables */
{ size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
{ size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, dctx->fseEntropy);
if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
if (LLtype != set_repeat) dctx->LLTptr = dctx->LLTable;
ip += llhSize;
}
{ size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, dctx->fseEntropy);
if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
if (OFtype != set_repeat) dctx->OFTptr = dctx->OFTable;
ip += ofhSize;
}
{ size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, dctx->fseEntropy);
if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
if (MLtype != set_repeat) dctx->MLTptr = dctx->MLTable;
ip += mlhSize;
} }
@ -714,16 +741,13 @@ static size_t ZSTD_decompressSequences(
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
const BYTE* const litEnd = litPtr + dctx->litSize;
FSE_DTable* DTableLL = dctx->LLTable;
FSE_DTable* DTableML = dctx->MLTable;
FSE_DTable* DTableOffb = dctx->OffTable;
const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
int nbSeq;
/* Build Decoding Tables */
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize);
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
if (ZSTD_isError(seqHSize)) return seqHSize;
ip += seqHSize;
}
@ -733,10 +757,10 @@ static size_t ZSTD_decompressSequences(
seqState_t seqState;
dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
CHECK_E(BIT_initDStream(&(seqState.DStream), ip, iend-ip), corruption_detected);
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
nbSeq--;
@ -894,21 +918,6 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
}
/*! ZSTD_decompress_usingPreparedDCtx() :
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
ZSTD_copyDCtx(dctx, refDCtx);
ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
}
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@ -1120,7 +1129,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize;
}
@ -1172,7 +1181,6 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
return ZSTD_refDictContent(dctx, dict, dictSize);
}
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
CHECK_F(ZSTD_decompressBegin(dctx));
@ -1181,6 +1189,8 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
}
/* ====== ZSTD_DDict ====== */
struct ZSTD_DDict_s {
void* dict;
size_t dictSize;
@ -1239,10 +1249,17 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
}
}
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support sizeof on NULL */
return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize;
}
/*! ZSTD_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary
* Use dictionary without significant overhead. */
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_DDict* ddict)
@ -1250,9 +1267,9 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
#endif
return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext,
dst, dstCapacity,
src, srcSize);
ZSTD_refDCtx(dctx, ddict->refContext);
ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
}
@ -1265,7 +1282,8 @@ typedef enum { zdss_init, zdss_loadHeader,
/* *** Resource management *** */
struct ZSTD_DStream_s {
ZSTD_DCtx* zd;
ZSTD_DCtx* dctx;
ZSTD_DDict* ddict;
ZSTD_frameParams fParams;
ZSTD_dStreamStage stage;
char* inBuff;
@ -1277,12 +1295,9 @@ struct ZSTD_DStream_s {
size_t outStart;
size_t outEnd;
size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */
size_t lhSize;
ZSTD_customMem customMem;
void* dictContent;
size_t dictSize;
const void* dictSource;
void* legacyContext;
U32 previousLegacyVersion;
U32 legacyVersion;
@ -1306,8 +1321,8 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
if (zds==NULL) return NULL;
memset(zds, 0, sizeof(ZSTD_DStream));
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
zds->zd = ZSTD_createDCtx_advanced(customMem);
if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; }
zds->dctx = ZSTD_createDCtx_advanced(customMem);
if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; }
zds->stage = zdss_init;
zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
return zds;
@ -1317,10 +1332,10 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
{
if (zds==NULL) return 0; /* support free on null */
{ ZSTD_customMem const cMem = zds->customMem;
ZSTD_freeDCtx(zds->zd);
ZSTD_freeDCtx(zds->dctx);
ZSTD_freeDDict(zds->ddict);
ZSTD_free(zds->inBuff, cMem);
ZSTD_free(zds->outBuff, cMem);
ZSTD_free(zds->dictContent, cMem);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (zds->legacyContext)
ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion);
@ -1340,15 +1355,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
{
zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) { /* new dictionary */
if (dictSize > zds->dictSize) {
ZSTD_free(zds->dictContent, zds->customMem);
zds->dictContent = ZSTD_malloc(dictSize, zds->customMem);
if (zds->dictContent == NULL) return ERROR(memory_allocation);
}
memcpy(zds->dictContent, dict, dictSize);
zds->dictSize = dictSize;
}
ZSTD_freeDDict(zds->ddict);
zds->ddict = ZSTD_createDDict(dict, dictSize);
if (zds->ddict == NULL) return ERROR(memory_allocation);
zds->legacyVersion = 0;
zds->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix;
@ -1359,6 +1368,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
return ZSTD_initDStream_usingDict(zds, NULL, 0);
}
size_t ZSTD_resetDStream(ZSTD_DStream* zds)
{
zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
zds->legacyVersion = 0;
zds->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix;
}
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
ZSTD_DStreamParameter_e paramType, unsigned paramValue)
{
@ -1373,7 +1391,8 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
{
return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize + zds->dictSize;
if (zds==NULL) return 0; /* support sizeof on NULL */
return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize;
}
@ -1415,7 +1434,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) {
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
zds->dictContent, zds->dictSize));
zds->ddict->dict, zds->ddict->dictSize));
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
} else {
@ -1437,11 +1456,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
/* Consume header */
ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize);
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_prefix */
CHECK_F(ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size));
{ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd);
CHECK_F(ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size));
ZSTD_refDCtx(zds->dctx, zds->ddict->refContext);
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
{ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size));
} }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
@ -1467,15 +1486,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
/* pass-through */
case zdss_read:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (neededInSize==0) { /* end of frame */
zds->stage = zdss_init;
someMoreWork = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
@ -1491,7 +1510,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
}
case zdss_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
@ -1501,8 +1520,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
{ const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
zds->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
@ -1534,7 +1553,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
/* result */
input->pos += (size_t)(ip-istart);
output->pos += (size_t)(op-ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (!nextSrcSizeHint) { /* frame fully decoded */
if (zds->outEnd == zds->outStart) { /* output fully flushed */
if (zds->hostageByte) {
@ -1549,7 +1568,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
}
return 1;
}
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); /* preload header of next block */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/
return nextSrcSizeHint;

65
lib/zstd.h
View File

@ -48,40 +48,41 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void);
* Simple API
***************************************/
/*! ZSTD_compress() :
Compresses `src` buffer into already allocated `dst`.
Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
@return : the number of bytes written into `dst` (<= `dstCapacity),
@return : compressed size written into `dst` (<= `dstCapacity),
or an error code if it fails (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
/*! ZSTD_getDecompressedSize() :
* @return : decompressed size as a 64-bits value _if known_, 0 otherwise.
* note 1 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 2 : decompressed size is an optional field, that may not be present.
* When `return==0`, data to decompress can have any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application may rely on its own implied limits.
* (For example, application data could be necessarily cut into blocks <= 16 KB).
* note 3 : decompressed size could be wrong or intentionally modified !
* Always ensure result fits within application's authorized limits !
* Each application can set its own limits.
* note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_decompress() :
`compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail.
`dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ).
If originalSize is unknown, and if there is no implied application-specific limitations,
it's preferable to use streaming mode to decompress data.
`compressedSize` : must be the _exact_ size of a single compressed frame.
`dstCapacity` is an upper bound of originalSize.
If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/*! ZSTD_getDecompressedSize() :
* 'src' is the start of a zstd compressed frame.
* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
* When `return==0`, data to decompress could be any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
* (For example, data may be necessarily cut into blocks <= 16 KB).
* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
* note 3 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
* Always ensure result fits within application's authorized limits.
* Each application can set its own limits.
* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*====== Helper functions ======*/
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
@ -351,14 +352,18 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_sizeofCCtx() :
* Gives the amount of memory used by a given ZSTD_CCtx */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
/*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
ZSTD_parameters params, ZSTD_customMem customMem);
/*! ZSTD_sizeofCCtx() :
* Gives the amount of memory used by a given ZSTD_CCtx */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
/*! ZSTD_sizeof_CDict() :
* Gives the amount of memory used by a given ZSTD_sizeof_CDict */
ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
/*! ZSTD_getParams() :
* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
@ -398,10 +403,14 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
* Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_sizeofDCtx() :
/*! ZSTD_sizeof_DCtx() :
* Gives the amount of memory used by a given ZSTD_DCtx */
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
/*! ZSTD_sizeof_DDict() :
* Gives the amount of memory used by a given ZSTD_DDict */
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
/* ******************************************************************
* Advanced Streaming functions
@ -412,7 +421,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
@ -423,6 +433,7 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);

5
programs/datagen.c
View File

@ -20,10 +20,9 @@
/*-************************************
* Dependencies
**************************************/
#include <stdlib.h> /* malloc */
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* FILE, fwrite, fprintf */
#include <string.h> /* memcpy */
#include <errno.h> /* errno */
#include "mem.h" /* U32 */
@ -176,7 +175,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
BYTE ldt[LTSIZE]; /* literals distribution table */
/* init */
if (buff==NULL) { fprintf(stderr, "datagen: error: %s \n", strerror(errno)); exit(1); }
if (buff==NULL) { perror("datagen"); exit(1); }
if (litProba<=0.0) litProba = matchProba / 4.5;
memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */
RDG_fillLiteralDistrib(ldt, litProba);

83
programs/fileio.c
View File

@ -119,8 +119,6 @@ static clock_t g_time = 0;
***************************************/
static U32 g_overwrite = 0;
void FIO_overwriteMode(void) { g_overwrite=1; }
static U32 g_maxWLog = 23;
void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
static U32 g_dictIDFlag = 1;
@ -167,7 +165,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
return f;
}
/* `dstFileName must` be non-NULL */
/** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* FIO_openDstFile(const char* dstFileName)
{
FILE* f;
@ -250,14 +250,12 @@ typedef struct {
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
void* dictBuffer;
size_t dictBufferSize;
ZSTD_CStream* cctx;
FILE* dstFile;
FILE* srcFile;
} cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName)
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel)
{
cRess_t ress;
memset(&ress, 0, sizeof(ress));
@ -271,19 +269,27 @@ static cRess_t FIO_createCResources(const char* dictFileName)
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory");
/* dictionary */
ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName);
{ void* dictBuffer;
size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName);
if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictBuffSize);
params.fParams.contentSizeFlag = 1;
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, 0);
if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
} }
free(dictBuffer);
}
return ress;
}
static void FIO_freeCResources(cRess_t ress)
{
size_t errorCode;
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
errorCode = ZSTD_freeCStream(ress.cctx);
if (ZSTD_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZSTD_CStream : %s", ZSTD_getErrorName(errorCode));
ZSTD_freeCStream(ress.cctx); /* never fails */
}
@ -293,8 +299,7 @@ static void FIO_freeCResources(cRess_t ress)
* 1 : missing or pb opening srcFileName
*/
static int FIO_compressFilename_internal(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
const char* dstFileName, const char* srcFileName)
{
FILE* const srcFile = ress.srcFile;
FILE* const dstFile = ress.dstFile;
@ -303,17 +308,9 @@ static int FIO_compressFilename_internal(cRess_t ress,
U64 const fileSize = UTIL_getFileSize(srcFileName);
/* init */
{ ZSTD_parameters params = ZSTD_getParams(cLevel, fileSize, ress.dictBufferSize);
params.fParams.contentSizeFlag = 1;
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
if ((g_maxWLog) && (params.cParams.windowLog > g_maxWLog)) {
params.cParams.windowLog = g_maxWLog;
params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize);
{ size_t const resetError = ZSTD_resetCStream(ress.cctx, fileSize);
if (ZSTD_isError(resetError)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(resetError));
}
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZSTD_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(errorCode));
} }
/* Main compression loop */
while (1) {
@ -367,8 +364,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
* 1 : missing or pb opening srcFileName
*/
static int FIO_compressFilename_srcFile(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
const char* dstFileName, const char* srcFileName)
{
int result;
@ -380,10 +376,10 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
ress.srcFile = FIO_openSrcFile(srcFileName);
if (!ress.srcFile) return 1; /* srcFile could not be opened */
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName);
fclose(ress.srcFile);
if ((g_removeSrcFile) && (!result)) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); }
if (g_removeSrcFile && !result) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */
return result;
}
@ -393,17 +389,16 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
* 1 : pb
*/
static int FIO_compressFilename_dstFile(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
const char* dstFileName, const char* srcFileName)
{
int result;
ress.dstFile = FIO_openDstFile(dstFileName);
if (ress.dstFile==0) return 1;
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel);
result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName);
if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; }
if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */
if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */
return result;
}
@ -414,8 +409,8 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
{
clock_t const start = clock();
cRess_t const ress = FIO_createCResources(dictFileName);
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel);
int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName);
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
@ -433,7 +428,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
size_t dfnSize = FNSPACE;
char* dstFileName = (char*)malloc(FNSPACE);
size_t const suffixSize = suffix ? strlen(suffix) : 0;
cRess_t ress = FIO_createCResources(dictFileName);
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel);
/* init */
if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");
@ -445,8 +440,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
ress.dstFile = stdout;
SET_BINARY_MODE(stdout);
for (u=0; u<nbFiles; u++)
missed_files += FIO_compressFilename_srcFile(ress, stdoutmark,
inFileNamesTable[u], compressionLevel);
missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u]);
if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
} else {
unsigned u;
@ -455,8 +449,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
strcpy(dstFileName, inFileNamesTable[u]);
strcat(dstFileName, suffix);
missed_files += FIO_compressFilename_dstFile(ress, dstFileName,
inFileNamesTable[u], compressionLevel);
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u]);
} }
/* Close & Free */
@ -480,8 +473,6 @@ typedef struct {
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
void* dictBuffer;
size_t dictBufferSize;
ZSTD_DStream* dctx;
FILE* dstFile;
} dRess_t;
@ -501,7 +492,12 @@ static dRess_t FIO_createDResources(const char* dictFileName)
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
/* dictionary */
ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName);
{ void* dictBuffer;
size_t const dictBufferSize = FIO_loadFile(&dictBuffer, dictFileName);
size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize);
if (ZSTD_isError(initError)) EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError));
free(dictBuffer);
}
return ress;
}
@ -512,7 +508,6 @@ static void FIO_freeDResources(dRess_t ress)
if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", ZSTD_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
}
@ -601,7 +596,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
size_t readSize;
U32 storedSkips = 0;
ZSTD_initDStream_usingDict(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
ZSTD_resetDStream(ress.dctx);
/* Header loading (optional, saves one loop) */
{ size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */

6
programs/fileio.h
View File

@ -8,7 +8,8 @@
*/
#pragma once
#ifndef FILEIO_H_23981798732
#define FILEIO_H_23981798732
#if defined (__cplusplus)
extern "C" {
@ -32,7 +33,6 @@ extern "C" {
***************************************/
void FIO_overwriteMode(void);
void FIO_setNotificationLevel(unsigned level);
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setChecksumFlag(unsigned checksumFlag);
@ -70,3 +70,5 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
#if defined (__cplusplus)
}
#endif
#endif /* FILEIO_H_23981798732 */

3
programs/util.h
View File

@ -274,7 +274,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
return nbFiles;
}
#elif (defined(__unix__) || defined(__unix) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) /* snprintf, opendir */
#elif (defined(__APPLE__) && defined(__MACH__)) || \
((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */
# define UTIL_HAS_CREATEFILELIST
# include <dirent.h> /* opendir, readdir */
# include <errno.h>

4
programs/zstdcli.c
View File

@ -262,7 +262,7 @@ int main(int argCount, char** argv)
if (!strcmp(argument, "--verbose")) { displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; FIO_setMaxWLog(0); continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
@ -334,8 +334,10 @@ int main(int argCount, char** argv)
/* destination file name */
case 'o': nextArgumentIsOutFileName=1; argument++; break;
#ifdef UTIL_HAS_CREATEFILELIST
/* recursive */
case 'r': recursive=1; argument++; break;
#endif
#ifndef ZSTD_NOBENCH
/* Benchmark */

2
tests/datagencli.c
View File

@ -9,7 +9,7 @@
/*-************************************
* Includes
* Dependencies
**************************************/
#include "util.h" /* Compiler options */
#include <stdio.h> /* fprintf, stderr */

6
tests/fullbench.c
View File

@ -121,13 +121,12 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co
}
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 tableRepeatFlag, const void* src, size_t srcSize);
extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize);
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */
int nbSeq;
(void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, 0, buff2, g_cSize);
return ZSTD_decodeSeqHeaders(g_zdc, &nbSeq, buff2, g_cSize);
}
@ -289,6 +288,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
}
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
ip += ZSTD_blockHeaderSize; /* skip block header */
ZSTD_decompressBegin(g_zdc);
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */
g_cSize = iend-ip;
memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */

2
tests/fuzzer.c
View File

@ -57,7 +57,7 @@ static U32 g_displayLevel = 2;
if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 150 / 1000;
static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_displayClock = 0;

2
tests/playTests.sh
View File

@ -205,7 +205,7 @@ $MD5SUM dirTestDict/* > tmph1
$ZSTD -f --rm dirTestDict/* -D tmpDictC
$ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default
if [[ "$OSTYPE" == "darwin"* ]]; then
$ECHO "test skipped on OS-X" # not compatible with OS-X's md5
$ECHO "md5sum -c not supported on OS-X : test skipped" # not compatible with OS-X's md5
else
$MD5SUM -c tmph1
fi

12
tests/zstreamtest.c
View File

@ -209,7 +209,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Byte-by-byte decompression test */
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
{ size_t r = 1;
{ /* skippable frame */
size_t r = 1;
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
inBuff.src = compressedBuffer;
outBuff.dst = decodedBuffer;
@ -221,9 +222,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error;
}
/* normal frame */
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
r=1;
while (r) { /* normal frame */
while (r) {
inBuff.size = inBuff.pos + 1;
outBuff.size = outBuff.pos + 1;
r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
@ -322,6 +324,8 @@ _output_error:
}
/* ====== Fuzzer tests ====== */
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
{
const BYTE* b1 = (const BYTE*)buf1;
@ -413,8 +417,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
/* states full reset (unsynchronized) */
/* some issues only happen when reusing states in a specific sequence of parameters */
/* states full reset (deliberately not synchronized) */
/* some issues can only happen when reusing states */
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); }
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); }

209
zstd_compression_format.md
View File

@ -16,7 +16,7 @@ Distribution of this document is unlimited.
### Version
0.2.0 (22/07/16)
0.2.2 (14/09/16)
Introduction
@ -1049,15 +1049,23 @@ by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
#### Bitstream
All sequences are stored in a single bitstream, read _backward_.
It is therefore necessary to know the bitstream size,
which is deducted from compressed block size.
FSE bitstreams are read in reverse direction than written. In zstd,
the compressor writes bits forward into a block and the decompressor
must read the bitstream _backwards_.
The last useful bit of the stream is followed by an end-bit-flag.
Highest bit of last byte is this flag.
It does not belong to the useful part of the bitstream.
Therefore, last byte has 0-7 useful bits.
Note that it also means that last byte cannot be `0`.
To find the start of the bitstream it is therefore necessary to
know the offset of the last byte of the block which can be found
by counting `Block_Size` bytes after the block header.
After writing the last bit containing information, the compressor
writes a single `1`-bit and then fills the byte with 0-7 `0` bits of
padding. The last byte of the compressed bitstream cannot be `0` for
that reason.
When decompressing, the last byte containing the padding is the first
byte to read. The decompressor needs to skip 0-7 initial `0`-bits and
the first `1`-bit it occurs. Afterwards, the useful part of the bitstream
begins.
##### Starting states
@ -1164,9 +1172,192 @@ __`Content`__ : The rest of the dictionary is its content.
[compressed blocks]: #the-format-of-compressed_block
Appendix A - Decoding tables for predefined codes
-------------------------------------------------
This appendix contains FSE decoding tables for the predefined literal length, match length, and offset
codes. The tables have been constructed using the algorithm as given above in the
"from normalized distribution to decoding tables" chapter. The tables here can be used as examples
to crosscheck that an implementation implements the decoding table generation algorithm correctly.
#### Literal Length Code:
| State | Symbol | Number_Of_Bits | Base |
| ----- | ------ | -------------- | ---- |
| 0 | 0 | 4 | 0 |
| 1 | 0 | 4 | 16 |
| 2 | 1 | 5 | 32 |
| 3 | 3 | 5 | 0 |
| 4 | 4 | 5 | 0 |
| 5 | 6 | 5 | 0 |
| 6 | 7 | 5 | 0 |
| 7 | 9 | 5 | 0 |
| 8 | 10 | 5 | 0 |
| 9 | 12 | 5 | 0 |
| 10 | 14 | 6 | 0 |
| 11 | 16 | 5 | 0 |
| 12 | 18 | 5 | 0 |
| 13 | 19 | 5 | 0 |
| 14 | 21 | 5 | 0 |
| 15 | 22 | 5 | 0 |
| 16 | 24 | 5 | 0 |
| 17 | 25 | 5 | 32 |
| 18 | 26 | 5 | 0 |
| 19 | 27 | 6 | 0 |
| 20 | 29 | 6 | 0 |
| 21 | 31 | 6 | 0 |
| 22 | 0 | 4 | 32 |
| 23 | 1 | 4 | 0 |
| 24 | 2 | 5 | 0 |
| 25 | 4 | 5 | 32 |
| 26 | 5 | 5 | 0 |
| 27 | 7 | 5 | 32 |
| 28 | 8 | 5 | 0 |
| 29 | 10 | 5 | 32 |
| 30 | 11 | 5 | 0 |
| 31 | 13 | 6 | 0 |
| 32 | 16 | 5 | 32 |
| 33 | 17 | 5 | 0 |
| 34 | 19 | 5 | 32 |
| 35 | 20 | 5 | 0 |
| 36 | 22 | 5 | 32 |
| 37 | 23 | 5 | 0 |
| 38 | 25 | 4 | 0 |
| 39 | 25 | 4 | 16 |
| 40 | 26 | 5 | 32 |
| 41 | 28 | 6 | 0 |
| 42 | 30 | 6 | 0 |
| 43 | 0 | 4 | 48 |
| 44 | 1 | 4 | 16 |
| 45 | 2 | 5 | 32 |
| 46 | 3 | 5 | 32 |
| 47 | 5 | 5 | 32 |
| 48 | 6 | 5 | 32 |
| 49 | 8 | 5 | 32 |
| 50 | 9 | 5 | 32 |
| 51 | 11 | 5 | 32 |
| 52 | 12 | 5 | 32 |
| 53 | 15 | 6 | 0 |
| 54 | 17 | 5 | 32 |
| 55 | 18 | 5 | 32 |
| 56 | 20 | 5 | 32 |
| 57 | 21 | 5 | 32 |
| 58 | 23 | 5 | 32 |
| 59 | 24 | 5 | 32 |
| 60 | 35 | 6 | 0 |
| 61 | 34 | 6 | 0 |
| 62 | 33 | 6 | 0 |
| 63 | 32 | 6 | 0 |
#### Match Length Code:
| State | Symbol | Number_Of_Bits | Base |
| ----- | ------ | -------------- | ---- |
| 0 | 0 | 6 | 0 |
| 1 | 1 | 4 | 0 |
| 2 | 2 | 5 | 32 |
| 3 | 3 | 5 | 0 |
| 4 | 5 | 5 | 0 |
| 5 | 6 | 5 | 0 |
| 6 | 8 | 5 | 0 |
| 7 | 10 | 6 | 0 |
| 8 | 13 | 6 | 0 |
| 9 | 16 | 6 | 0 |
| 10 | 19 | 6 | 0 |
| 11 | 22 | 6 | 0 |
| 12 | 25 | 6 | 0 |
| 13 | 28 | 6 | 0 |
| 14 | 31 | 6 | 0 |
| 15 | 33 | 6 | 0 |
| 16 | 35 | 6 | 0 |
| 17 | 37 | 6 | 0 |
| 18 | 39 | 6 | 0 |
| 19 | 41 | 6 | 0 |
| 20 | 43 | 6 | 0 |
| 21 | 45 | 6 | 0 |
| 22 | 1 | 4 | 16 |
| 23 | 2 | 4 | 0 |
| 24 | 3 | 5 | 32 |
| 25 | 4 | 5 | 0 |
| 26 | 6 | 5 | 32 |
| 27 | 7 | 5 | 0 |
| 28 | 9 | 6 | 0 |
| 29 | 12 | 6 | 0 |
| 30 | 15 | 6 | 0 |
| 31 | 18 | 6 | 0 |
| 32 | 21 | 6 | 0 |
| 33 | 24 | 6 | 0 |
| 34 | 27 | 6 | 0 |
| 35 | 30 | 6 | 0 |
| 36 | 32 | 6 | 0 |
| 37 | 34 | 6 | 0 |
| 38 | 36 | 6 | 0 |
| 39 | 38 | 6 | 0 |
| 40 | 40 | 6 | 0 |
| 41 | 42 | 6 | 0 |
| 42 | 44 | 6 | 0 |
| 43 | 1 | 4 | 32 |
| 44 | 1 | 4 | 48 |
| 45 | 2 | 4 | 16 |
| 46 | 4 | 5 | 32 |
| 47 | 5 | 5 | 32 |
| 48 | 7 | 5 | 32 |
| 49 | 8 | 5 | 32 |
| 50 | 11 | 6 | 0 |
| 51 | 14 | 6 | 0 |
| 52 | 17 | 6 | 0 |
| 53 | 20 | 6 | 0 |
| 54 | 23 | 6 | 0 |
| 55 | 26 | 6 | 0 |
| 56 | 29 | 6 | 0 |
| 57 | 52 | 6 | 0 |
| 58 | 51 | 6 | 0 |
| 59 | 50 | 6 | 0 |
| 60 | 49 | 6 | 0 |
| 61 | 48 | 6 | 0 |
| 62 | 47 | 6 | 0 |
| 63 | 46 | 6 | 0 |
#### Offset Code:
| State | Symbol | Number_Of_Bits | Base |
| ----- | ------ | -------------- | ---- |
| 0 | 0 | 5 | 0 |
| 1 | 6 | 4 | 0 |
| 2 | 9 | 5 | 0 |
| 3 | 15 | 5 | 0 |
| 4 | 21 | 5 | 0 |
| 5 | 3 | 5 | 0 |
| 6 | 7 | 4 | 0 |
| 7 | 12 | 5 | 0 |
| 8 | 18 | 5 | 0 |
| 9 | 23 | 5 | 0 |
| 10 | 5 | 5 | 0 |
| 11 | 8 | 4 | 0 |
| 12 | 14 | 5 | 0 |
| 13 | 20 | 5 | 0 |
| 14 | 2 | 5 | 0 |
| 15 | 7 | 4 | 16 |
| 16 | 11 | 5 | 0 |
| 17 | 17 | 5 | 0 |
| 18 | 22 | 5 | 0 |
| 19 | 4 | 5 | 0 |
| 20 | 8 | 4 | 16 |
| 21 | 13 | 5 | 0 |
| 22 | 19 | 5 | 0 |
| 23 | 1 | 5 | 0 |
| 24 | 6 | 4 | 16 |
| 25 | 10 | 5 | 0 |
| 26 | 16 | 5 | 0 |
| 27 | 28 | 5 | 0 |
| 28 | 27 | 5 | 0 |
| 29 | 26 | 5 | 0 |
| 30 | 25 | 5 | 0 |
| 31 | 24 | 5 | 0 |
Version changes
---------------
- 0.2.2 : added predefined codes, by Johannes Rudolph
- 0.2.1 : clarify field names, by Przemyslaw Skibinski
- 0.2.0 : numerous format adjustments for zstd v0.8
- 0.1.2 : limit Huffman tree depth to 11 bits