lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-24 17:21:03 +03:00
Code Activity

implemented ZSTD_dParam_getBounds()

Browse Source 
and ZSTD_DCtx_setParameter()
This commit is contained in:
Yann Collet
2018-12-04 15:35:37 -08:00
parent 34e146f548
commit aec945f0dc
4 changed files with 186 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/compress/zstd_compress.c
View File

@@ -128,7 +128,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx)
#ifdef ZSTD_MULTITHREAD
return ZSTDMT_sizeof_CCtx(cctx->mtctx);
#else
(void) cctx;
(void)cctx;
return 0;
#endif
}

91
lib/decompress/zstd_decompress.c
View File

@@ -1256,6 +1256,80 @@ size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
return 0;
}
ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
{
ZSTD_bounds bounds = { 0, 0, 0 };
switch(dParam) {
case ZSTD_d_windowLogMax:
bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN;
bounds.upperBound = ZSTD_WINDOWLOG_MAX;
return bounds;
case ZSTD_d_format:
bounds.lowerBound = (int)ZSTD_f_zstd1;
bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
return bounds;
default:
bounds.error = ERROR(parameter_unsupported);
return bounds;
}
assert(0);
bounds.error = ERROR(parameter_unsupported);
return bounds;
}
/* ZSTD_dParam_withinBounds:
* @return 1 if value is within dParam bounds,
* 0 otherwise */
static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
{
ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam);
if (ZSTD_isError(bounds.error)) return 0;
if (value < bounds.lowerBound) return 0;
if (value > bounds.upperBound) return 0;
return 1;
}
#define CHECK_DBOUNDS(p,v) { \
if (!ZSTD_dParam_withinBounds(p, v)) \
return ERROR(parameter_outOfBound); \
}
size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
{
if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
switch(dParam) {
case ZSTD_d_windowLogMax:
CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);
dctx->maxWindowSize = 1 << value;
return 0;
case ZSTD_d_format:
CHECK_DBOUNDS(ZSTD_d_format, value);
dctx->format = (ZSTD_format_e)value;
return 0;
default:
return ERROR(parameter_unsupported);
}
assert(0); /* should be unreachable */
return ERROR(parameter_unsupported);
}
size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
{
if ( (reset == ZSTD_reset_session_only)
|| (reset == ZSTD_reset_session_and_parameters) ) {
(void)ZSTD_initDStream(dctx);
}
if ( (reset == ZSTD_reset_parameters)
|| (reset == ZSTD_reset_session_and_parameters) ) {
if (dctx->streamStage != zdss_init)
return ERROR(stage_wrong);
dctx->format = ZSTD_f_zstd1;
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
}
return 0;
}
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
{
@@ -1578,20 +1652,3 @@ size_t ZSTD_decompressStream_simpleArgs (
*srcPos = input.pos;
return cErr;
}
size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
{
if ( (reset == ZSTD_reset_session_only)
|| (reset == ZSTD_reset_session_and_parameters) ) {
(void)ZSTD_initDStream(dctx);
}
if ( (reset == ZSTD_reset_parameters)
|| (reset == ZSTD_reset_session_and_parameters) ) {
if (dctx->streamStage != zdss_init)
return ERROR(stage_wrong);
dctx->format = ZSTD_f_zstd1;
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
}
return 0;
}

18
lib/zstd.h
View File

@@ -640,7 +640,7 @@ typedef struct {
* otherwise they will either trigger an error or be automatically clamped.
* @return : a structure, ZSTD_bounds, which contains
* - an error status field, which must be tested using ZSTD_isError()
* - both lower and upper bounds, inclusive
* - lower and upper bounds, both inclusive
*/
ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
@@ -834,16 +834,16 @@ typedef enum {
* - an error status field, which must be tested using ZSTD_isError()
* - both lower and upper bounds, inclusive
*/
ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); /* not implemented yet */
ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
/*! ZSTD_DCtx_setParameter() :
* Set one compression parameter, selected by enum ZSTD_dParameter.
* All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
* Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
* Setting a parameter is only possible during frame initialization (before starting decompression).
* @return : an error code (which can be tested using ZSTD_isError()).
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
*/
ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); /* not implemented yet */
ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
/*! ZSTD_DCtx_loadDictionary() :
@@ -895,7 +895,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
/*! ZSTD_DCtx_reset() :
* Return a DCtx to clean state.
* Session and parameters can be reset jointly or separately
* Session and parameters can be reset jointly or separately.
* Parameters can only be reset when no active frame is being decompressed.
* @return : 0, or an error code, which can be tested with ZSTD_isError()
*/
@@ -1003,7 +1003,7 @@ typedef enum {
ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
* Useful to save 4 bytes per generated frame.
* Decoder cannot recognise automatically this format, requiring instructions. */
* Decoder cannot recognise automatically this format, requiring this instruction. */
} ZSTD_format_e;
typedef enum {
@@ -1440,6 +1440,12 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* pre
*/
ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
/* ZSTD_d_format
* experimental parameter,
* allowing selection between ZSTD_format_e input compression formats
*/
#define ZSTD_d_format ZSTD_d_experimentalParam1
/*! ZSTD_DCtx_setFormat() :
* Instruct the decoder context about what kind of data to decode next.
* This instruction is mandatory to decode data without a fully-formed header,

135
tests/fuzzer.c
View File

@@ -309,7 +309,6 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
void* const compressedBuffer = malloc(compressedBufferSize);
void* const decodedBuffer = malloc(CNBuffSize);
ZSTD_DCtx* dctx = ZSTD_createDCtx();
int testResult = 0;
U32 testNb=0;
size_t cSize;
@@ -381,13 +380,19 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
if (r != CNBuffSize) goto _output_error; }
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
if (r != CNBuffSize) goto _output_error;
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL);
if (r != CNBuffSize) goto _output_error; }
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL);
if (r != CNBuffSize) goto _output_error;
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
@@ -544,8 +549,8 @@ static int basicUnitTests(U32 seed, double compressibility)
/* this test is really too long, and should be made faster */
DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0);
size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 1; /* ensure U32 overflow protection is triggered */
ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
size_t cnb;
assert(cctx != NULL);
params.fParams.contentSizeFlag = 0;
@@ -775,7 +780,9 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Dictionary and CCtx Duplication tests */
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
static const size_t dictSize = 551;
assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
@@ -859,6 +866,7 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_freeCCtx(ctxOrig);
ZSTD_freeCCtx(ctxDuplicated);
ZSTD_freeDCtx(dctx);
}
/* Dictionary and dictBuilder tests */
@@ -903,7 +911,7 @@ static int basicUnitTests(U32 seed, double compressibility)
coverParams.nbThreads = 4;
dictSize = ZDICT_optimizeTrainFromBuffer_cover(
dictBuffer, dictBufferCapacity,
CNBuffer, samplesSizes, nbSamples,
CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
&coverParams);
if (ZDICT_isError(dictSize)) goto _output_error;
}
@@ -948,11 +956,14 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
@@ -981,11 +992,14 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
@@ -1034,11 +1048,14 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK (unknown)\n");
DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
@@ -1052,19 +1069,24 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
{
const size_t ret = ZSTD_decompress_usingDict(
dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
"\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) goto _output_error;
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
const size_t ret = ZSTD_decompress_usingDict(
dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
"\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
goto _output_error;
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
@@ -1133,10 +1155,12 @@ static int basicUnitTests(U32 seed, double compressibility)
*/
{ size_t dSize;
BYTE data[1024];
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
ZSTD_dlm_byRef, ZSTD_dct_auto,
cParams, ZSTD_defaultCMem);
assert(dctx != NULL); assert(cdict != NULL);
memset(data, 'x', sizeof(data));
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
data, sizeof(data), cdict);
@@ -1145,6 +1169,7 @@ static int basicUnitTests(U32 seed, double compressibility)
dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
ZSTD_freeDCtx(dctx);
}
DISPLAYLEVEL(3, "OK \n");
@@ -1300,9 +1325,43 @@ static int basicUnitTests(U32 seed, double compressibility)
}
}
/* advanced parameters for decompression */
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
assert(dctx != NULL);
DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
{ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
CHECK(bounds.error);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
{ size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
if (!ZSTD_isError(sr)) goto _output_error;
}
{ ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
if (!ZSTD_isError(bounds.error)) goto _output_error;
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
{ size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
if (!ZSTD_isError(sr)) goto _output_error;
}
{ size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
if (!ZSTD_isError(sr)) goto _output_error;
}
DISPLAYLEVEL(3, "OK \n");
ZSTD_freeDCtx(dctx);
}
/* custom formats tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
assert(dctx != NULL); assert(cctx != NULL);
/* basic block compression */
DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
@@ -1339,13 +1398,16 @@ static int basicUnitTests(U32 seed, double compressibility)
}
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
}
/* block API tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
static const size_t dictSize = 65 KB;
static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
size_t cSize2;
assert(cctx != NULL); assert(dctx != NULL);
/* basic block compression */
DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
@@ -1363,13 +1425,14 @@ static int basicUnitTests(U32 seed, double compressibility)
/* very long stream of block compression */
DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
CHECK( ZSTD_compressBegin(cctx, -99) ); /* we just want to quickly overflow internal U32 index */
CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
{ U64 const toCompress = 5000000000ULL; /* > 4 GB */
U64 compressed = 0;
while (compressed < toCompress) {
size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
assert(blockCSize != 0);
if (ZSTD_isError(blockCSize)) goto _output_error;
compressed += blockCSize;
}
}
@@ -1407,8 +1470,8 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
}
ZSTD_freeDCtx(dctx);
/* long rle test */
{ size_t sampleSize = 0;