lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-24 17:21:03 +03:00
Code Activity

fixed bug in dubt

Browse Source 
the chain of unsorted candidates could grow beyond lowLimit.
This commit is contained in:
Yann Collet
2017-12-29 17:04:37 +01:00
parent f36da5b4d9
commit 64482c2c97
3 changed files with 228 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
lib/compress/zstd_compress.c
View File

@@ -752,9 +752,9 @@ static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t blockSize1,
size_t const windowSize2 = MAX(1, (size_t)MIN(((U64)1 << cParams2.windowLog), pledgedSrcSize));
size_t const blockSize2 = MIN(ZSTD_BLOCKSIZE_MAX, windowSize2);
size_t const neededBufferSize2 = (buffPol2==ZSTDb_buffered) ? windowSize2 + blockSize2 : 0;
DEBUGLOG(4, "ZSTD_sufficientBuff: windowSize2=%u from wlog=%u",
DEBUGLOG(4, "ZSTD_sufficientBuff: is windowSize2=%u <= wlog1=%u",
(U32)windowSize2, cParams2.windowLog);
DEBUGLOG(4, "ZSTD_sufficientBuff: blockSize2 %u <=? blockSize1 %u",
DEBUGLOG(4, "ZSTD_sufficientBuff: is blockSize2=%u <= blockSize1=%u",
(U32)blockSize2, (U32)blockSize1);
return (blockSize2 <= blockSize1) /* seqStore space depends on blockSize */
& (neededBufferSize2 <= bufferSize1);
@@ -777,10 +777,12 @@ static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1,
* reuse CCtx without reset (note : requires no dictionary) */
static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize)
{
U32 const end = (U32)(cctx->nextSrc - cctx->base);
size_t const endT = (size_t)(cctx->nextSrc - cctx->base);
U32 const end = (U32)endT;
size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize));
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
DEBUGLOG(4, "ZSTD_continueCCtx");
DEBUGLOG(4, "ZSTD_continueCCtx: re-use context in place");
assert(endT < (3U<<30));
cctx->blockSize = blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */
cctx->appliedParams = params;
@@ -912,12 +914,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->entropy->offcode_repeatMode = FSE_repeat_none;
zc->entropy->matchlength_repeatMode = FSE_repeat_none;
zc->entropy->litlength_repeatMode = FSE_repeat_none;
zc->nextToUpdate = 1;
zc->nextSrc = NULL;
zc->base = NULL;
zc->dictBase = NULL;
zc->dictLimit = 0;
zc->lowLimit = 0;
zc->dictLimit = 0;
zc->nextToUpdate = 1;
zc->nextSrc = NULL;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->seqStore.rep[i] = repStartValue[i]; }
zc->hashLog3 = hashLog3;
zc->optState.litLengthSum = 0;
@@ -925,7 +927,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ptr = zc->entropy + 1;
/* opt parser space */
if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) {
if ((params.cParams.strategy == ZSTD_btopt) | (params.cParams.strategy == ZSTD_btultra)) {
DEBUGLOG(4, "reserving optimal parser space");
assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */
zc->optState.litFreq = (U32*)ptr;
@@ -1643,9 +1645,12 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
ssPtr->longLengthID = 0;
}
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_internal : dstCapacity = %u", (U32)dstCapacity);
DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u) (dictLimit=%u, nextToUpdate=%u)",
(U32)dstCapacity, zc->dictLimit, zc->nextToUpdate);
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1)
return 0; /* don't even attempt compression below a certain srcSize */
ZSTD_resetSeqStore(&(zc->seqStore));
@@ -1737,13 +1742,17 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
cctx->dictLimit -= correction;
if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0;
else cctx->nextToUpdate -= correction;
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x\n", correction, cctx->lowLimit);
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, cctx->lowLimit);
}
/* enforce maxDist */
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
if (cctx->dictLimit < cctx->lowLimit)
DEBUGLOG(2, "ZSTD_compress_frameChunk : update dictLimit from %u to %u ",
cctx->dictLimit, cctx->lowLimit);
if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
if (cctx->nextToUpdate < cctx->lowLimit) cctx->nextToUpdate = cctx->lowLimit;
}
{ size_t cSize = ZSTD_compressBlock_internal(cctx,
@@ -1846,8 +1855,9 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
/* Check if blocks follow each other */
if (src != cctx->nextSrc) {
/* not contiguous */
size_t const distanceFromBase = (size_t)(cctx->nextSrc - cctx->base);
DEBUGLOG(5, "ZSTD_compressContinue_internal: non contiguous blocks, new segment starts at %u",
cctx->dictLimit);
cctx->lowLimit = cctx->dictLimit;
assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */
cctx->dictLimit = (U32)distanceFromBase;
@@ -1879,6 +1889,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (U32)srcSize);
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */);
}
@@ -2144,10 +2155,10 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
DEBUGLOG(4, "ZSTD_compressBegin_usingDict");
DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (U32)dictSize);
return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
}
@@ -2275,6 +2286,7 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, co
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel)
{
DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (U32)srcSize);
return ZSTD_compress_usingDict(ctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel);
}
@@ -2885,7 +2897,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (params.nbThreads > 1) {
if (cctx->mtctx == NULL || (params.nbThreads != ZSTDMT_getNbThreads(cctx->mtctx))) {
DEBUGLOG(4, "ZSTD_compress_generic: creating new mtctx for nbThreads=%u (previous: %u)",
params.nbThreads, ZSTDMT_getNbThreads(cctx->mtctx));
params.nbThreads, (unsigned)ZSTDMT_getNbThreads(cctx->mtctx));
ZSTDMT_freeCCtx(cctx->mtctx);
cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbThreads, cctx->customMem);
if (cctx->mtctx == NULL) return ERROR(memory_allocation);

22
lib/compress/zstd_lazy.c
View File

@@ -32,12 +32,12 @@ void ZSTD_updateDUBT(ZSTD_CCtx* zc,
U32 const target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u ",
idx, target);
if (idx != target)
DEBUGLOG(2, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)",
idx, target, zc->dictLimit);
assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */
(void)iend;
assert(idx >= zc->dictLimit); /* condition for valid base+idx */
for ( ; idx < target ; idx++) {
size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */
@@ -79,10 +79,11 @@ static void ZSTD_insertDUBT1(ZSTD_CCtx* zc,
U32 dummy32; /* to be nullified at the end */
U32 const windowLow = zc->lowLimit;
DEBUGLOG(8, "ZSTD_insertDUBT1 (%u)", current);
DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)",
current, dictLimit, windowLow);
assert(current >= btLow);
if (extDict && (current < dictLimit)) { /* do not sort candidates in _extDict (simplification, for easier ZSTD_count, detrimental to compression ratio in streaming mode) */
if (extDict && (current < dictLimit)) { /* candidates in _extDict are not sorted (simplification, for easier ZSTD_count, detrimental to compression ratio in streaming mode) */
*largerPtr = *smallerPtr = 0;
return;
}
@@ -147,11 +148,13 @@ static size_t ZSTD_insertBtAndFindBestMatch (
const BYTE* const base = zc->base;
U32 const current = (U32)(ip-base);
U32 const windowLow = zc->lowLimit;
U32* const bt = zc->chainTable;
U32 const btLog = zc->appliedParams.cParams.chainLog - 1;
U32 const btMask = (1 << btLog) - 1;
U32 const btLow = (btMask >= current) ? 0 : current - btMask;
U32 const unsortLimit = MAX(btLow, windowLow);
U32* nextCandidate = bt + 2*(matchIndex&btMask);
U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1;
@@ -162,7 +165,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
assert(ip <= iend-8); /* required for h calculation */
/* reach end of unsorted candidates list */
while ( (matchIndex > btLow)
while ( (matchIndex > unsortLimit)
&& (*unsortedMark == ZSTD_DUBT_UNSORTED)
&& (nbCandidates > 1) ) {
DEBUGLOG(8, "ZSTD_insertBtAndFindBestMatch: candidate %u is unsorted",
@@ -175,11 +178,11 @@ static size_t ZSTD_insertBtAndFindBestMatch (
nbCandidates --;
}
if ( (matchIndex > btLow)
if ( (matchIndex > unsortLimit)
&& (*unsortedMark==ZSTD_DUBT_UNSORTED) ) {
DEBUGLOG(8, "ZSTD_insertBtAndFindBestMatch: nullify last unsorted candidate %u",
matchIndex);
*nextCandidate = *unsortedMark = 0; /* nullify last candidate if it's still unsorted (note : detrimental to compression ratio) */
*nextCandidate = *unsortedMark = 0; /* nullify next candidate if it's still unsorted (note : simplification, detrimental to compression ratio, beneficial for speed) */
}
/* batch sort stacked candidates */
@@ -188,7 +191,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1;
U32 const nextCandidateIdx = *nextCandidateIdxPtr;
ZSTD_insertDUBT1(zc, matchIndex, iend,
nbCandidates, btLow, extDict);
nbCandidates, unsortLimit, extDict);
matchIndex = nextCandidateIdx;
nbCandidates++;
}
@@ -199,7 +202,6 @@ static size_t ZSTD_insertBtAndFindBestMatch (
const U32 dictLimit = zc->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const U32 windowLow = zc->lowLimit;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
U32 matchEndIdx = current+8+1;

371
tests/fuzzer.c
View File

@@ -286,91 +286,91 @@ static int basicUnitTests(U32 seed, double compressibility)
RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
/* Basic tests */
DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName : ", testNb++);
DISPLAYLEVEL(3, "test%3i : ZSTD_getErrorName : ", testNb++);
{ const char* errorString = ZSTD_getErrorName(0);
DISPLAYLEVEL(4, "OK : %s \n", errorString);
DISPLAYLEVEL(3, "OK : %s \n", errorString);
}
DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++);
DISPLAYLEVEL(3, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++);
{ const char* errorString = ZSTD_getErrorName(499);
DISPLAYLEVEL(4, "OK : %s \n", errorString);
DISPLAYLEVEL(3, "OK : %s \n", errorString);
}
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
if (cctx==NULL) goto _output_error;
CHECKPLUS(r, ZSTD_compressCCtx(cctx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, 1),
cSize=r );
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : size of cctx for level 1 : ", testNb++);
DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
DISPLAYLEVEL(4, "%u bytes \n", (U32)cctxSize);
DISPLAYLEVEL(3, "%u bytes \n", (U32)cctxSize);
}
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(4, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
{ unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
if (rSize != CNBuffSize) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
{ unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
if (rSize != CNBuffSize) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
{ size_t u;
for (u=0; u<CNBuffSize; u++) {
if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with null DDict : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
if (!ZSTD_isError(r)) goto _output_error;
if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
if (!ZSTD_isError(r)) goto _output_error;
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%di : check CCtx size after compressing empty input : ", testNb++);
DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
if (ZSTD_isError(r)) goto _output_error;
if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%di : re-use CCtx with expanding block size : ", testNb++);
DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
@@ -385,11 +385,11 @@ static int basicUnitTests(U32 seed, double compressibility)
}
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
/* Static CCtx tests */
#define STATIC_CCTX_LEVEL 3
DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
{ size_t const staticCCtxSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
void* const staticCCtxBuffer = malloc(staticCCtxSize);
size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
@@ -404,57 +404,57 @@ static int basicUnitTests(U32 seed, double compressibility)
{ ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
ZSTD_DCtx* staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
{ size_t const r = ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL);
if (ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : simple compression test with static CCtx : ", testNb++);
DISPLAYLEVEL(3, "test%3i : simple compression test with static CCtx : ", testNb++);
CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL),
cSize=r );
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n",
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
(U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : simple decompression test with static DCtx : ", testNb++);
DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
{ size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
{ size_t u;
for (u=0; u<CNBuffSize; u++) {
if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u])
goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
{ size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
{ size_t const r = ZSTD_compressBegin(staticCCtx, 1);
if (ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CStream for small level %u : ", testNb++, 1);
DISPLAYLEVEL(3, "test%3i : init CStream for small level %u : ", testNb++, 1);
{ size_t const r = ZSTD_initCStream(staticCCtx, 1);
if (ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CStream with dictionary (should fail) : ", testNb++);
DISPLAYLEVEL(3, "test%3i : init CStream with dictionary (should fail) : ", testNb++);
{ size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : init DStream (should fail) : ", testNb++);
DISPLAYLEVEL(3, "test%3i : init DStream (should fail) : ", testNb++);
{ size_t const r = ZSTD_initDStream(staticDCtx);
if (ZSTD_isError(r)) goto _output_error; }
{ ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
@@ -462,53 +462,52 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
if (!ZSTD_isError(r)) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
}
free(staticCCtxBuffer);
free(staticDCtxBuffer);
}
/* ZSTDMT simple MT compression test */
DISPLAYLEVEL(4, "test%3i : create ZSTDMT CCtx : ", testNb++);
DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
{ ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);
if (mtctx==NULL) {
DISPLAY("mtctx : mot enough memory, aborting \n");
testResult = 1;
goto _end;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress %u bytes with 2 threads : ", testNb++, (U32)CNBuffSize);
DISPLAYLEVEL(3, "test%3i : compress %u bytes with 2 threads : ", testNb++, (U32)CNBuffSize);
CHECKPLUS(r, ZSTDMT_compressCCtx(mtctx,
compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize,
1),
cSize=r );
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
{ unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
if (rSize != CNBuffSize) {
DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (U32)rSize, (U32)CNBuffSize);
goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
{ size_t u;
for (u=0; u<CNBuffSize; u++) {
if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress -T2 with checksum : ", testNb++);
DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, 0);
params.fParams.checksumFlag = 1;
params.fParams.contentSizeFlag = 1;
@@ -518,19 +517,19 @@ static int basicUnitTests(U32 seed, double compressibility)
NULL, params, 3 /*overlapRLog*/),
cSize=r );
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
ZSTDMT_freeCCtx(mtctx);
}
/* Simple API multiframe test */
DISPLAYLEVEL(4, "test%3i : compress multiple frames : ", testNb++);
DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);
{ size_t off = 0;
int i;
int const segs = 4;
@@ -552,53 +551,53 @@ static int basicUnitTests(U32 seed, double compressibility)
}
cSize = off;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : get decompressed size of multiple frames : ", testNb++);
DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++);
{ unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
if (r != CNBuffSize / 2) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress multiple frames : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
{ CHECK_V(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
if (r != CNBuffSize / 2) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
/* Dictionary and CCtx Duplication tests */
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
static const size_t dictSize = 551;
DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
cSize = 0;
CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, compressedBufferSize,
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
cSize += r);
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
CNBuffer, dictSize),
if (r != CNBuffSize - dictSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
{ size_t const cSizeOrig = cSize;
cSize = 0;
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, compressedBufferSize,
@@ -606,25 +605,25 @@ static int basicUnitTests(U32 seed, double compressibility)
cSize += r);
if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
CNBuffer, dictSize),
if (r != CNBuffSize - dictSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with DDict : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++);
{ ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
if (r != CNBuffSize - dictSize) goto _output_error;
DISPLAYLEVEL(4, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict));
DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict));
ZSTD_freeDDict(ddict);
}
DISPLAYLEVEL(4, "test%3i : decompress with static DDict : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
{ size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
void* ddictBuffer = malloc(ddictBufferSize);
if (ddictBuffer == NULL) goto _output_error;
@@ -633,10 +632,10 @@ static int basicUnitTests(U32 seed, double compressibility)
if (r != CNBuffSize - dictSize) goto _output_error;
}
free(ddictBuffer);
DISPLAYLEVEL(4, "OK (size of static DDict : %u) \n", (U32)ddictBufferSize);
DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (U32)ddictBufferSize);
}
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
{ size_t const testSize = CNBuffSize / 3;
{ ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize);
p.fParams.contentSizeFlag = 1;
@@ -651,7 +650,7 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
ZSTD_freeCCtx(ctxOrig);
ZSTD_freeCCtx(ctxDuplicated);
@@ -673,79 +672,79 @@ static int basicUnitTests(U32 seed, double compressibility)
goto _output_error;
}
DISPLAYLEVEL(4, "test%3i : dictBuilder : ", testNb++);
DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++);
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
CNBuffer, samplesSizes, nbSamples);
if (ZDICT_isError(dictSize)) goto _output_error;
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (U32)dictSize);
DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
dictID = ZDICT_getDictID(dictBuffer, dictSize);
if (dictID==0) goto _output_error;
DISPLAYLEVEL(4, "OK : %u \n", dictID);
DISPLAYLEVEL(3, "OK : %u \n", dictID);
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize,
dictBuffer, dictSize, 4);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : retrieve dictID from dictionary : ", testNb++);
DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++);
{ U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++);
DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
{ U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef);
DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
DISPLAYLEVEL(3, "OK : %u \n", (U32)estimatedSize);
}
DISPLAYLEVEL(4, "test%3i : compress with CDict ", testNb++);
DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
ZSTD_dlm_byRef, ZSTD_dm_auto,
cParams, ZSTD_defaultCMem);
DISPLAYLEVEL(4, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict));
DISPLAYLEVEL(3, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict));
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, cdict);
ZSTD_freeCDict(cdict);
if (ZSTD_isError(cSize)) goto _output_error;
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++);
DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
{ U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with static CDict : ", testNb++);
DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
void* const cdictBuffer = malloc(cdictSize);
@@ -767,9 +766,9 @@ static int basicUnitTests(U32 seed, double compressibility)
} }
free(cdictBuffer);
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
{ ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, ZSTD_defaultCMem);
@@ -778,23 +777,23 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_freeCDict(cdict);
if (ZSTD_isError(cSize)) goto _output_error;
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : try retrieving contentSize from frame : ", testNb++);
DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++);
{ U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
}
DISPLAYLEVEL(4, "OK (unknown)\n");
DISPLAYLEVEL(3, "OK (unknown)\n");
DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
{ ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
p.fParams.noDictIDFlag = 1;
cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
@@ -802,42 +801,42 @@ static int basicUnitTests(U32 seed, double compressibility)
dictBuffer, dictSize, p);
if (ZSTD_isError(cSize)) goto _output_error;
}
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
dictBuffer, dictSize),
if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : dictionary containing only header should return error : ", testNb++);
DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
{
const size_t ret = ZSTD_decompress_usingDict(
dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
"\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
if (cdict==NULL) goto _output_error;
ZSTD_freeCDict(cdict);
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
if (cdict!=NULL) goto _output_error;
ZSTD_freeCDict(cdict);
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++);
{
size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@@ -845,9 +844,9 @@ static int basicUnitTests(U32 seed, double compressibility)
cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dm_auto);
if (!ZSTD_isError(ret)) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++);
{
size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@@ -855,9 +854,9 @@ static int basicUnitTests(U32 seed, double compressibility)
cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dm_rawContent);
if (ZSTD_isError(ret)) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary with non-default repcodes : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
CNBuffer, samplesSizes, nbSamples);
@@ -899,7 +898,7 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
ZSTD_freeCCtx(cctx);
free(dictBuffer);
@@ -924,7 +923,7 @@ static int basicUnitTests(U32 seed, double compressibility)
goto _output_error;
}
DISPLAYLEVEL(4, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
memset(&params, 0, sizeof(params));
params.d = 1 + (FUZ_rand(&seed) % 16);
@@ -933,26 +932,26 @@ static int basicUnitTests(U32 seed, double compressibility)
CNBuffer, samplesSizes, nbSamples,
params);
if (ZDICT_isError(dictSize)) goto _output_error;
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (U32)dictSize);
DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
dictID = ZDICT_getDictID(dictBuffer, dictSize);
if (dictID==0) goto _output_error;
DISPLAYLEVEL(4, "OK : %u \n", dictID);
DISPLAYLEVEL(3, "OK : %u \n", dictID);
DISPLAYLEVEL(4, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
memset(&params, 0, sizeof(params));
params.steps = 4;
optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize,
CNBuffer, samplesSizes,
nbSamples / 4, &params);
if (ZDICT_isError(optDictSize)) goto _output_error;
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)optDictSize);
DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (U32)optDictSize);
DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
dictID = ZDICT_getDictID(dictBuffer, optDictSize);
if (dictID==0) goto _output_error;
DISPLAYLEVEL(4, "OK : %u \n", dictID);
DISPLAYLEVEL(3, "OK : %u \n", dictID);
ZSTD_freeCCtx(cctx);
free(dictBuffer);
@@ -960,20 +959,20 @@ static int basicUnitTests(U32 seed, double compressibility)
}
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
if (!ZSTD_isError(r)) goto _output_error;
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1;
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
/* content size verification test */
DISPLAYLEVEL(4, "test%3i : Content size verification : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t const srcSize = 5000;
size_t const wrongSrcSize = (srcSize + 1000);
@@ -983,7 +982,7 @@ static int basicUnitTests(U32 seed, double compressibility)
{ size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
if (!ZSTD_isError(result)) goto _output_error;
if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(result));
DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result));
}
ZSTD_freeCCtx(cctx);
}
@@ -993,7 +992,7 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
/* basic block compression */
DISPLAYLEVEL(4, "test%3i : magic-less format test : ", testNb++);
DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
@@ -1002,15 +1001,15 @@ static int basicUnitTests(U32 seed, double compressibility)
if (in.pos != in.size) goto _output_error;
cSize = out.pos;
}
DISPLAYLEVEL(4, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize);
DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize);
DISPLAYLEVEL(4, "test%3i : decompress normally (should fail) : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++);
{ size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(decodeResult));
DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult));
}
DISPLAYLEVEL(4, "test%3i : decompress with magic-less instruction : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress with magic-less instruction : ", testNb++);
ZSTD_DCtx_reset(dctx);
CHECK( ZSTD_DCtx_setFormat(dctx, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
@@ -1019,7 +1018,7 @@ static int basicUnitTests(U32 seed, double compressibility)
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
if (out.pos != inputSize) goto _output_error;
DISPLAYLEVEL(4, "OK : regenerated %u bytes \n", (U32)out.pos);
DISPLAYLEVEL(3, "OK : regenerated %u bytes \n", (U32)out.pos);
}
ZSTD_freeCCtx(cctx);
@@ -1032,21 +1031,21 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t cSize2;
/* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
CHECK( ZSTD_compressBegin(cctx, 5) );
CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
CHECK( ZSTD_decompressBegin(dctx) );
{ CHECK_V(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
if (r != blockSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
/* dictionary block compression */
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
@@ -1056,16 +1055,16 @@ static int basicUnitTests(U32 seed, double compressibility)
cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
(char*)CNBuffer+dictSize+2*blockSize, blockSize);
if (ZSTD_isError(cSize2)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
{ CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
if (r != blockSize) goto _output_error; }
ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
{ CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, CNBuffSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
if (r != blockSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
ZSTD_freeCCtx(cctx);
}
@@ -1073,7 +1072,7 @@ static int basicUnitTests(U32 seed, double compressibility)
/* long rle test */
{ size_t sampleSize = 0;
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1;
@@ -1083,21 +1082,21 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_isError(cSize)) goto _output_error;
{ CHECK_V(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
if (regenSize!=sampleSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
}
/* All zeroes test (test bug #137) */
#define ZEROESLENGTH 100
DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
memset(CNBuffer, 0, ZEROESLENGTH);
{ CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
cSize = r; }
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
{ CHECK_V(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
if (r != ZEROESLENGTH) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
/* nbSeq limit test */
#define _3BYTESTESTLENGTH 131000
@@ -1124,18 +1123,18 @@ static int basicUnitTests(U32 seed, double compressibility)
((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
} } }
DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
{ CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
CNBuffer, _3BYTESTESTLENGTH, 19) );
cSize = r; }
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
{ CHECK_V(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
if (r != _3BYTESTESTLENGTH) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(4, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
{ /* Train a dictionary on low characters */
size_t dictSize = 16 KB;
@@ -1168,25 +1167,25 @@ static int basicUnitTests(U32 seed, double compressibility)
free(dictBuffer);
free(samplesSizes);
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
/* findFrameCompressedSize on skippable frames */
DISPLAYLEVEL(4, "test%3i : frame compressed size of skippable frame : ", testNb++);
DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++);
{ const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
size_t const frameSrcSize = 13;
if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
/* error string tests */
DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++);
DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(3, "OK \n");
_end:
free(CNBuffer);
@@ -1369,6 +1368,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
}
/* successful decompression test */
DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb);
{ size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
@@ -1379,6 +1379,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
free(sampleBuffer); /* no longer useful after this point */
/* truncated src decompression test */
DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb);
{ size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
size_t const tooSmallSize = cSize - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
@@ -1390,6 +1391,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
}
/* too small dst decompression test */
DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb);
if (sampleSize > 3) {
size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
size_t const tooSmallSize = sampleSize - missing;
@@ -1425,6 +1427,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
} } }
/* decompress noisy source */
DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb);
{ U32 const endMark = 0xA9B1C3D6;
memcpy(dstBuffer+sampleSize, &endMark, 4);
{ size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
@@ -1436,8 +1439,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
} } } /* noisy src decompression test */
/*===== Streaming compression test, scattered segments and dictionary =====*/
/*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
int const cLevel = (FUZ_rand(&lseed) %
@@ -1450,10 +1453,13 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
testNb, (U32)maxTestSize, cLevel, (U32)dictSize);
if (FUZ_rand(&lseed) & 0xF) {
CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
} else {
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
@@ -1491,6 +1497,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
}
/* streaming decompression test */
DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
/* ensure memory requirement is good enough (should always be true) */
{ ZSTD_frameHeader zfh;
CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_frameHeaderSize_max),
@@ -1633,7 +1640,7 @@ int main(int argc, const char** argv)
case 'v':
argument++;
g_displayLevel = 4;
g_displayLevel++;
break;
case 'q':