lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-01 09:47:01 +03:00
Code Activity

[lib] Add ZSTD_c_deterministicRefPrefix

Browse Source 
This flag forces zstd to always load the prefix in ext-dict mode, even
if it happens to be contiguous, to force determinism. It also applies to
dictionaries that are re-processed.

A determinism test case is also added, which fails without
`ZSTD_c_deterministicRefPrefix` and passes with it set.

Question: Should this be the default behavior? It isn't in this PR.
This commit is contained in:
Nick Terrell
2021-05-05 12:18:47 -07:00
parent 2d10544b84
commit 172b4b6ac4
7 changed files with 109 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
tests/fuzzer.c
View File

@ -766,6 +766,50 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++);
{
size_t const testSize = 1024;
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
char* dict = (char*)malloc(2 * testSize);
int ldmEnabled, level;
RDG_genBuffer(dict, testSize, 0.5, 0.5, seed);
RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
memcpy(dict + testSize, CNBuffer, testSize);
for (level = 1; level <= 5; ++level) {
for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) {
size_t cSize0;
XXH64_hash_t compressedChecksum0;
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1));
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize);
CHECK_Z(cSize);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize));
cSize0 = cSize;
compressedChecksum0 = XXH64(compressedBuffer, cSize, 0);
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize);
CHECK_Z(cSize);
if (cSize != cSize0) goto _output_error;
if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error;
}
}
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
free(dict);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();