mirror of
https://github.com/facebook/zstd.git
synced 2025-07-30 22:23:13 +03:00
[libzstd] Fix decompression dictionary bugs and clean up initialization
Bugs: * `ZSTD_DCtx_refPrefix()` didn't clear the dictionary after the first use. Fix and add a test case. * `ZSTD_DCtx_reset()` always cleared the dictionary. Fix and add a test case. * After calling `ZSTD_resetDStream()` you could no longer load a dictionary, since the stage was set to `zdss_loadHeader`. Fix and add a test case. Cleanup: * Make `ZSTD_initDStream*()` and `ZSTD_resetDStream()` wrap the new advanced API, and add test cases. * Document the equivalent of these functions in the advanced API and document the unstable functions as deprecated.
This commit is contained in:
@ -106,6 +106,7 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
|
|||||||
dctx->ddictLocal = NULL;
|
dctx->ddictLocal = NULL;
|
||||||
dctx->dictEnd = NULL;
|
dctx->dictEnd = NULL;
|
||||||
dctx->ddictIsCold = 0;
|
dctx->ddictIsCold = 0;
|
||||||
|
dctx->dictUsesRemaining = 0;
|
||||||
dctx->inBuff = NULL;
|
dctx->inBuff = NULL;
|
||||||
dctx->inBuffSize = 0;
|
dctx->inBuffSize = 0;
|
||||||
dctx->outBuffSize = 0;
|
dctx->outBuffSize = 0;
|
||||||
@ -147,13 +148,20 @@ ZSTD_DCtx* ZSTD_createDCtx(void)
|
|||||||
return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
|
return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void ZSTD_clearDict(ZSTD_DCtx* dctx)
|
||||||
|
{
|
||||||
|
ZSTD_freeDDict(dctx->ddictLocal);
|
||||||
|
dctx->ddictLocal = NULL;
|
||||||
|
dctx->ddict = NULL;
|
||||||
|
dctx->dictUsesRemaining = 0;
|
||||||
|
}
|
||||||
|
|
||||||
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
|
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
|
||||||
{
|
{
|
||||||
if (dctx==NULL) return 0; /* support free on NULL */
|
if (dctx==NULL) return 0; /* support free on NULL */
|
||||||
RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
|
RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
|
||||||
{ ZSTD_customMem const cMem = dctx->customMem;
|
{ ZSTD_customMem const cMem = dctx->customMem;
|
||||||
ZSTD_freeDDict(dctx->ddictLocal);
|
ZSTD_clearDict(dctx);
|
||||||
dctx->ddictLocal = NULL;
|
|
||||||
ZSTD_free(dctx->inBuff, cMem);
|
ZSTD_free(dctx->inBuff, cMem);
|
||||||
dctx->inBuff = NULL;
|
dctx->inBuff = NULL;
|
||||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
|
||||||
@ -786,9 +794,22 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
|
||||||
|
{
|
||||||
|
if (dctx->dictUsesRemaining == 0) {
|
||||||
|
ZSTD_clearDict(dctx);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (dctx->dictUsesRemaining < 0) {
|
||||||
|
return dctx->ddict;
|
||||||
|
}
|
||||||
|
--dctx->dictUsesRemaining;
|
||||||
|
return dctx->ddict;
|
||||||
|
}
|
||||||
|
|
||||||
size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, dctx->ddict);
|
return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -1235,14 +1256,13 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
|
|||||||
ZSTD_dictContentType_e dictContentType)
|
ZSTD_dictContentType_e dictContentType)
|
||||||
{
|
{
|
||||||
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
||||||
ZSTD_freeDDict(dctx->ddictLocal);
|
ZSTD_clearDict(dctx);
|
||||||
if (dict && dictSize >= 8) {
|
if (dict && dictSize >= 8) {
|
||||||
dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
|
dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
|
||||||
RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
|
RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
|
||||||
} else {
|
dctx->ddict = dctx->ddictLocal;
|
||||||
dctx->ddictLocal = NULL;
|
dctx->dictUsesRemaining = -1;
|
||||||
}
|
}
|
||||||
dctx->ddict = dctx->ddictLocal;
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1258,7 +1278,9 @@ size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
|
|||||||
|
|
||||||
size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
|
size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
|
||||||
{
|
{
|
||||||
return ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType);
|
FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType));
|
||||||
|
dctx->dictUsesRemaining = 1;
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize)
|
size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize)
|
||||||
@ -1273,8 +1295,7 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz
|
|||||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
|
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
|
||||||
{
|
{
|
||||||
DEBUGLOG(4, "ZSTD_initDStream_usingDict");
|
DEBUGLOG(4, "ZSTD_initDStream_usingDict");
|
||||||
zds->streamStage = zdss_init;
|
FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) );
|
||||||
zds->noForwardProgress = 0;
|
|
||||||
FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
|
FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
|
||||||
return ZSTD_FRAMEHEADERSIZE_PREFIX;
|
return ZSTD_FRAMEHEADERSIZE_PREFIX;
|
||||||
}
|
}
|
||||||
@ -1283,7 +1304,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
|
|||||||
size_t ZSTD_initDStream(ZSTD_DStream* zds)
|
size_t ZSTD_initDStream(ZSTD_DStream* zds)
|
||||||
{
|
{
|
||||||
DEBUGLOG(4, "ZSTD_initDStream");
|
DEBUGLOG(4, "ZSTD_initDStream");
|
||||||
return ZSTD_initDStream_usingDict(zds, NULL, 0);
|
return ZSTD_initDStream_usingDDict(zds, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ZSTD_initDStream_usingDDict() :
|
/* ZSTD_initDStream_usingDDict() :
|
||||||
@ -1291,9 +1312,9 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
|
|||||||
* this function cannot fail */
|
* this function cannot fail */
|
||||||
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
|
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
|
||||||
{
|
{
|
||||||
size_t const initResult = ZSTD_initDStream(dctx);
|
FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) );
|
||||||
dctx->ddict = ddict;
|
FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) );
|
||||||
return initResult;
|
return ZSTD_FRAMEHEADERSIZE_PREFIX;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ZSTD_resetDStream() :
|
/* ZSTD_resetDStream() :
|
||||||
@ -1301,11 +1322,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
|
|||||||
* this function cannot fail */
|
* this function cannot fail */
|
||||||
size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
|
size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
|
||||||
{
|
{
|
||||||
DEBUGLOG(4, "ZSTD_resetDStream");
|
FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only));
|
||||||
dctx->streamStage = zdss_loadHeader;
|
|
||||||
dctx->lhSize = dctx->inPos = dctx->outStart = dctx->outEnd = 0;
|
|
||||||
dctx->legacyVersion = 0;
|
|
||||||
dctx->hostageByte = 0;
|
|
||||||
return ZSTD_FRAMEHEADERSIZE_PREFIX;
|
return ZSTD_FRAMEHEADERSIZE_PREFIX;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1313,7 +1330,11 @@ size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
|
|||||||
size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
|
size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
|
||||||
{
|
{
|
||||||
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
||||||
dctx->ddict = ddict;
|
ZSTD_clearDict(dctx);
|
||||||
|
if (ddict) {
|
||||||
|
dctx->ddict = ddict;
|
||||||
|
dctx->dictUsesRemaining = -1;
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1394,11 +1415,13 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
|
|||||||
{
|
{
|
||||||
if ( (reset == ZSTD_reset_session_only)
|
if ( (reset == ZSTD_reset_session_only)
|
||||||
|| (reset == ZSTD_reset_session_and_parameters) ) {
|
|| (reset == ZSTD_reset_session_and_parameters) ) {
|
||||||
(void)ZSTD_initDStream(dctx);
|
dctx->streamStage = zdss_init;
|
||||||
|
dctx->noForwardProgress = 0;
|
||||||
}
|
}
|
||||||
if ( (reset == ZSTD_reset_parameters)
|
if ( (reset == ZSTD_reset_parameters)
|
||||||
|| (reset == ZSTD_reset_session_and_parameters) ) {
|
|| (reset == ZSTD_reset_session_and_parameters) ) {
|
||||||
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
|
||||||
|
ZSTD_clearDict(dctx);
|
||||||
dctx->format = ZSTD_f_zstd1;
|
dctx->format = ZSTD_f_zstd1;
|
||||||
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
|
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
|
||||||
}
|
}
|
||||||
@ -1481,7 +1504,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|||||||
{
|
{
|
||||||
case zdss_init :
|
case zdss_init :
|
||||||
DEBUGLOG(5, "stage zdss_init => transparent reset ");
|
DEBUGLOG(5, "stage zdss_init => transparent reset ");
|
||||||
ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
|
zds->streamStage = zdss_loadHeader;
|
||||||
|
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
|
||||||
|
zds->legacyVersion = 0;
|
||||||
|
zds->hostageByte = 0;
|
||||||
/* fall-through */
|
/* fall-through */
|
||||||
|
|
||||||
case zdss_loadHeader :
|
case zdss_loadHeader :
|
||||||
@ -1501,8 +1527,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|||||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
||||||
U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
|
U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
|
||||||
if (legacyVersion) {
|
if (legacyVersion) {
|
||||||
const void* const dict = zds->ddict ? ZSTD_DDict_dictContent(zds->ddict) : NULL;
|
ZSTD_DDict const* const ddict = ZSTD_getDDict(zds);
|
||||||
size_t const dictSize = zds->ddict ? ZSTD_DDict_dictSize(zds->ddict) : 0;
|
const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL;
|
||||||
|
size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0;
|
||||||
DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
|
DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
|
||||||
RETURN_ERROR_IF(zds->staticSize, memory_allocation,
|
RETURN_ERROR_IF(zds->staticSize, memory_allocation,
|
||||||
"legacy support is incompatible with static dctx");
|
"legacy support is incompatible with static dctx");
|
||||||
@ -1540,7 +1567,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|||||||
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
|
size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
|
||||||
if (cSize <= (size_t)(iend-istart)) {
|
if (cSize <= (size_t)(iend-istart)) {
|
||||||
/* shortcut : using single-pass mode */
|
/* shortcut : using single-pass mode */
|
||||||
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict);
|
size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, ZSTD_getDDict(zds));
|
||||||
if (ZSTD_isError(decompressedSize)) return decompressedSize;
|
if (ZSTD_isError(decompressedSize)) return decompressedSize;
|
||||||
DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
|
DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
|
||||||
ip = istart + cSize;
|
ip = istart + cSize;
|
||||||
@ -1553,7 +1580,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|||||||
|
|
||||||
/* Consume header (see ZSTDds_decodeFrameHeader) */
|
/* Consume header (see ZSTDds_decodeFrameHeader) */
|
||||||
DEBUGLOG(4, "Consume header");
|
DEBUGLOG(4, "Consume header");
|
||||||
FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
|
FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)));
|
||||||
|
|
||||||
if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
||||||
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
|
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
|
||||||
|
@ -123,6 +123,9 @@ struct ZSTD_DCtx_s
|
|||||||
const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
|
const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
|
||||||
U32 dictID;
|
U32 dictID;
|
||||||
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
|
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
|
||||||
|
int dictUsesRemaining; /* if == 1 : dictionary should be used once.
|
||||||
|
* if == 0 : dictionary should be forgotten now.
|
||||||
|
* if < 0 : dictionary should be used indefinitely. */
|
||||||
|
|
||||||
/* streaming */
|
/* streaming */
|
||||||
ZSTD_dStreamStage streamStage;
|
ZSTD_dStreamStage streamStage;
|
||||||
|
36
lib/zstd.h
36
lib/zstd.h
@ -722,7 +722,14 @@ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
|
|||||||
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
||||||
|
|
||||||
/*===== Streaming decompression functions =====*/
|
/*===== Streaming decompression functions =====*/
|
||||||
|
|
||||||
|
/* This function is redundant with the advanced API and equivalent to:
|
||||||
|
*
|
||||||
|
* ZSTD_DCtx_reset(zds);
|
||||||
|
* ZSTD_DCtx_refDDict(zds, NULL);
|
||||||
|
*/
|
||||||
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
||||||
|
|
||||||
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
|
||||||
|
|
||||||
ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
|
ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
|
||||||
@ -1672,9 +1679,32 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
|
|||||||
|
|
||||||
|
|
||||||
/*===== Advanced Streaming decompression functions =====*/
|
/*===== Advanced Streaming decompression functions =====*/
|
||||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
|
/**
|
||||||
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */
|
* This function is deprecated, and is equivalent to:
|
||||||
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
|
*
|
||||||
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
||||||
|
* ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
|
||||||
|
*
|
||||||
|
* note: no dictionary will be used if dict == NULL or dictSize < 8
|
||||||
|
*/
|
||||||
|
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
|
||||||
|
/**
|
||||||
|
* This function is deprecated, and is equivalent to:
|
||||||
|
*
|
||||||
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
||||||
|
* ZSTD_DCtx_refDDict(zds, ddict);
|
||||||
|
*
|
||||||
|
* note : ddict is referenced, it must outlive decompression session
|
||||||
|
*/
|
||||||
|
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
|
||||||
|
/**
|
||||||
|
* This function is deprecated, and is equivalent to:
|
||||||
|
*
|
||||||
|
* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
|
||||||
|
*
|
||||||
|
* re-use decompression parameters from previous init; saves dictionary loading
|
||||||
|
*/
|
||||||
|
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
|
||||||
|
|
||||||
|
|
||||||
/*********************************************************************
|
/*********************************************************************
|
||||||
|
@ -1464,6 +1464,41 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
}
|
}
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
|
||||||
|
{
|
||||||
|
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||||
|
ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
|
||||||
|
size_t ret;
|
||||||
|
/* We should succeed to decompress with the ddict. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||||
|
CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
|
||||||
|
CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||||
|
/* The ddict should presist across calls. */
|
||||||
|
CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||||
|
/* When we reset the context the ddict is cleared. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||||
|
ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
|
||||||
|
if (!ZSTD_isError(ret)) goto _output_error;
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
ZSTD_freeDDict(ddict);
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
|
||||||
|
{
|
||||||
|
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||||
|
size_t ret;
|
||||||
|
/* We should succeed to decompress with the prefix. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||||
|
CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
|
||||||
|
CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||||
|
/* The prefix should be cleared after the first compression. */
|
||||||
|
ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
|
||||||
|
if (!ZSTD_isError(ret)) goto _output_error;
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
|
DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
|
||||||
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
|
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
|
||||||
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
|
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
|
||||||
|
@ -723,6 +723,156 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
}
|
}
|
||||||
DISPLAYLEVEL(3, "OK\n");
|
DISPLAYLEVEL(3, "OK\n");
|
||||||
|
|
||||||
|
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
|
||||||
|
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) );
|
||||||
|
cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB));
|
||||||
|
CHECK_Z(cSize);
|
||||||
|
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++);
|
||||||
|
{
|
||||||
|
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||||
|
/* We should fail to decompress without a dictionary. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
|
||||||
|
if (!ZSTD_isError(ret)) goto _output_error;
|
||||||
|
}
|
||||||
|
/* We should succeed to decompress with the dictionary. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||||
|
CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
}
|
||||||
|
/* The dictionary should presist across calls. */
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
}
|
||||||
|
/* The dictionary should not be cleared by ZSTD_reset_session_only. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
}
|
||||||
|
/* When we reset the context the dictionary is cleared. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
|
||||||
|
if (!ZSTD_isError(ret)) goto _output_error;
|
||||||
|
}
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() wtih dictionary : ", testNb++);
|
||||||
|
{
|
||||||
|
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||||
|
/* We should succeed to decompress with the dictionary. */
|
||||||
|
ZSTD_resetDStream(dctx);
|
||||||
|
CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
}
|
||||||
|
/* The dictionary should not be cleared by ZSTD_resetDStream(). */
|
||||||
|
ZSTD_resetDStream(dctx);
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
}
|
||||||
|
/* The dictionary should be cleared by ZSTD_initDStream(). */
|
||||||
|
CHECK_Z( ZSTD_initDStream(dctx) );
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
|
||||||
|
if (!ZSTD_isError(ret)) goto _output_error;
|
||||||
|
}
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++);
|
||||||
|
{
|
||||||
|
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||||
|
ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
|
||||||
|
/* We should succeed to decompress with the ddict. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||||
|
CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
}
|
||||||
|
/* The ddict should presist across calls. */
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
}
|
||||||
|
/* When we reset the context the ddict is cleared. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
|
||||||
|
if (!ZSTD_isError(ret)) goto _output_error;
|
||||||
|
}
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
ZSTD_freeDDict(ddict);
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
|
||||||
|
{
|
||||||
|
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||||
|
/* We should succeed to decompress with the prefix. */
|
||||||
|
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
|
||||||
|
CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) );
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
}
|
||||||
|
/* The prefix should be cleared after the first compression. */
|
||||||
|
{ ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
|
||||||
|
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
|
size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
|
||||||
|
if (!ZSTD_isError(ret)) goto _output_error;
|
||||||
|
}
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++);
|
||||||
|
{
|
||||||
|
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||||
|
ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
|
||||||
|
size_t ret;
|
||||||
|
/* We should succeed to decompress with the dictionary. */
|
||||||
|
CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) );
|
||||||
|
CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
|
||||||
|
/* The dictionary should presist across calls. */
|
||||||
|
CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
|
||||||
|
/* We should succeed to decompress with the ddict. */
|
||||||
|
CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) );
|
||||||
|
CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
|
||||||
|
/* The ddict should presist across calls. */
|
||||||
|
CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
|
||||||
|
/* When we reset the context the ddict is cleared. */
|
||||||
|
CHECK_Z( ZSTD_initDStream(dctx) );
|
||||||
|
ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize);
|
||||||
|
if (!ZSTD_isError(ret)) goto _output_error;
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
ZSTD_freeDDict(ddict);
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
|
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
|
||||||
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
|
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
|
||||||
ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
|
ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
|
||||||
|
Reference in New Issue
Block a user