lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-07 06:23:00 +03:00
Code Activity

strengthened bufferless streaming decompression

Browse Source
This commit is contained in:
Yann Collet
2015-12-05 09:23:53 +01:00
parent 1c2ddba469
commit e47c4e5f8e
4 changed files with 49 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
lib/zstd_compress.c
View File

@@ -769,12 +769,12 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
/* init */ /* init */
ZSTD_resetSeqStore(seqStorePtr); ZSTD_resetSeqStore(seqStorePtr);
if (ip < base+4) if (ip < lowest+4)
{ {
hashTable[ZSTD_hashPtr(base+1, hBits, mls)] = 1; hashTable[ZSTD_hashPtr(lowest+1, hBits, mls)] = zc->dictLimit+1;
hashTable[ZSTD_hashPtr(base+2, hBits, mls)] = 2; hashTable[ZSTD_hashPtr(lowest+2, hBits, mls)] = zc->dictLimit+2;
hashTable[ZSTD_hashPtr(base+3, hBits, mls)] = 3; hashTable[ZSTD_hashPtr(lowest+3, hBits, mls)] = zc->dictLimit+3;
ip = base+4; ip = lowest+4;
} }
/* Main Search Loop */ /* Main Search Loop */
@@ -1535,6 +1535,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
const BYTE* anchor = istart; const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize; const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8; const BYTE* const ilimit = iend - 8;
const BYTE* const base = ctx->base + ctx->dictLimit;
size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE; size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
const U32 maxSearches = 1 << ctx->params.searchLog; const U32 maxSearches = 1 << ctx->params.searchLog;
@@ -1547,7 +1548,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* init */ /* init */
ZSTD_resetSeqStore(seqStorePtr); ZSTD_resetSeqStore(seqStorePtr);
if (((ip-ctx->base) - ctx->dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; if ((ip-base) < REPCODE_STARTVALUE) ip = base + REPCODE_STARTVALUE;
/* Match Loop */ /* Match Loop */
while (ip < ilimit) while (ip < ilimit)
@@ -1633,7 +1634,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* catch up */ /* catch up */
if (offset) if (offset)
{ {
while ((start>anchor) && (start>ctx->base+offset) && (start[-1] == start[-1-offset])) /* only search for offset within prefix */ while ((start>anchor) && (start>base+offset) && (start[-1] == start[-1-offset])) /* only search for offset within prefix */
{ start--; matchLength++; } { start--; matchLength++; }
offset_2 = offset_1; offset_1 = offset; offset_2 = offset_1; offset_1 = offset;
} }
@@ -1990,7 +1991,6 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
void* dst, size_t dstSize, void* dst, size_t dstSize,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
U32 adressOverflow = 0;
const BYTE* const ip = (const BYTE*) src; const BYTE* const ip = (const BYTE*) src;
/* Check if blocks follow each other */ /* Check if blocks follow each other */
@@ -2001,13 +2001,13 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
zc->lowLimit = zc->dictLimit; zc->lowLimit = zc->dictLimit;
zc->dictLimit = (U32)(zc->nextSrc - zc->base); zc->dictLimit = (U32)(zc->nextSrc - zc->base);
zc->dictBase = zc->base; zc->dictBase = zc->base;
if ((size_t)zc->base < delta) adressOverflow = zc->lowLimit;
zc->base -= delta; zc->base -= delta;
zc->nextToUpdate = zc->dictLimit; zc->nextToUpdate = zc->dictLimit;
if (zc->dictLimit - zc->lowLimit < 8) zc->lowLimit = zc->dictLimit; /* too small extDict */
} }
/* preemptive overflow correction */ /* preemptive overflow correction */
if (adressOverflow || (zc->lowLimit > (1<<30) )) if ((zc->base > ip) || (zc->lowLimit > (1<<30) ))
{ {
U32 correction = zc->lowLimit-1; U32 correction = zc->lowLimit-1;
ZSTD_reduceIndex(zc, correction); ZSTD_reduceIndex(zc, correction);

2
programs/Makefile
View File

@@ -281,7 +281,7 @@ valgrindTest: zstd datagen fuzzer fullbench zbufftest
./datagen -g64MB > tmp ./datagen -g64MB > tmp
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID) valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
@rm tmp @rm tmp
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1 valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -T1mn -t1
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1
valgrind --leak-check=yes --error-exitcode=1 ./zbufftest -T1mn valgrind --leak-check=yes --error-exitcode=1 ./zbufftest -T1mn

41
programs/fuzzer.c
View File

@@ -261,6 +261,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
BYTE* srcBuffer; BYTE* srcBuffer;
BYTE* cBuffer; BYTE* cBuffer;
BYTE* dstBuffer; BYTE* dstBuffer;
BYTE* mirrorBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog; size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog; size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize); size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
@@ -268,18 +269,22 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
U32 testNb = 0; U32 testNb = 0;
U32 coreSeed = seed, lseed = 0; U32 coreSeed = seed, lseed = 0;
ZSTD_CCtx* ctx; ZSTD_CCtx* ctx;
ZSTD_DCtx* dctx;
U32 startTime = FUZ_GetMilliStart(); U32 startTime = FUZ_GetMilliStart();
/* allocation */ /* allocation */
ctx = ZSTD_createCCtx(); ctx = ZSTD_createCCtx();
dctx= ZSTD_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize); dstBuffer = (BYTE*)malloc (dstBufferSize);
mirrorBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize); cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !dstBuffer || !cBuffer || !ctx, CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|| !dstBuffer || !mirrorBuffer || !cBuffer || !ctx || !dctx,
"Not enough memory, fuzzer tests cancelled"); "Not enough memory, fuzzer tests cancelled");
/* Create initial samples */ /* Create initial samples */
@@ -298,7 +303,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{ {
size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
size_t cSize, dSize, dSupSize, errorCode; size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n; U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
XXH64_state_t crc64; XXH64_state_t crc64;
U64 crcOrig, crcDest; U64 crcOrig, crcDest;
@@ -439,12 +444,13 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
} }
} }
/* Multi - segments compression test */ /* Streaming compression of scattered segments test */
XXH64_reset(&crc64, 0); XXH64_reset(&crc64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2; nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog; maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
totalTestSize = 0; totalTestSize = 0;
cSize = ZSTD_compressBegin(ctx, cBuffer, cBufferSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); cSize = ZSTD_compressBegin(ctx, cBuffer, cBufferSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
for (n=0; n<nbChunks; n++) for (n=0; n<nbChunks; n++)
@@ -457,25 +463,49 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) if (cBufferSize-cSize < ZSTD_compressBound(sampleSize))
/* avoid invalid dstBufferTooSmall */ /* avoid invalid dstBufferTooSmall */
break; break;
if (totalTestSize+sampleSize > maxTestSize) break;
errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize); errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode)); CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode; cSize += errorCode;
XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize); XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
totalTestSize += sampleSize; totalTestSize += sampleSize;
if (totalTestSize > maxTestSize) break;
} }
errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize); errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode)); CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode; cSize += errorCode;
crcOrig = XXH64_digest(&crc64); crcOrig = XXH64_digest(&crc64);
/* streaming decompression test */
errorCode = ZSTD_resetDCtx(dctx);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
{
size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
totalGenSize += genSize;
totalCSize += inSize;
}
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig)
errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);
} }
DISPLAY("\r%u fuzzer tests completed \n", testNb-1); DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
_cleanup: _cleanup:
ZSTD_freeCCtx(ctx); ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
free(cNoiseBuffer[0]); free(cNoiseBuffer[0]);
free(cNoiseBuffer[1]); free(cNoiseBuffer[1]);
free(cNoiseBuffer[2]); free(cNoiseBuffer[2]);
@@ -483,6 +513,7 @@ _cleanup:
free(cNoiseBuffer[4]); free(cNoiseBuffer[4]);
free(cBuffer); free(cBuffer);
free(dstBuffer); free(dstBuffer);
free(mirrorBuffer);
return result; return result;
_output_error: _output_error:

2
programs/zstdcli.c
View File

@@ -119,7 +119,7 @@ static int usage(const char* programName)
DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
DISPLAY( "Arguments :\n"); DISPLAY( "Arguments :\n");
DISPLAY( " -1 : Fast compression (default) \n"); DISPLAY( " -1 : Fast compression (default) \n");
DISPLAY( " -9 : High compression \n"); DISPLAY( " -19 : High compression \n");
DISPLAY( " -d : decompression (default for %s extension)\n", ZSTD_EXTENSION); DISPLAY( " -d : decompression (default for %s extension)\n", ZSTD_EXTENSION);
//DISPLAY( " -z : force compression\n"); //DISPLAY( " -z : force compression\n");
DISPLAY( " -f : overwrite output without prompting \n"); DISPLAY( " -f : overwrite output without prompting \n");