lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-07 06:23:00 +03:00
Code Activity

strengthened bufferless streaming decompression

Browse Source
This commit is contained in:
Yann Collet
2015-12-05 09:23:53 +01:00
parent 1c2ddba469
commit e47c4e5f8e
4 changed files with 49 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
lib/zstd_compress.c
View File

@@ -769,12 +769,12 @@ size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
/* init */
ZSTD_resetSeqStore(seqStorePtr);
if (ip < base+4)
if (ip < lowest+4)
{
hashTable[ZSTD_hashPtr(base+1, hBits, mls)] = 1;
hashTable[ZSTD_hashPtr(base+2, hBits, mls)] = 2;
hashTable[ZSTD_hashPtr(base+3, hBits, mls)] = 3;
ip = base+4;
hashTable[ZSTD_hashPtr(lowest+1, hBits, mls)] = zc->dictLimit+1;
hashTable[ZSTD_hashPtr(lowest+2, hBits, mls)] = zc->dictLimit+2;
hashTable[ZSTD_hashPtr(lowest+3, hBits, mls)] = zc->dictLimit+3;
ip = lowest+4;
}
/* Main Search Loop */
@@ -1535,6 +1535,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
const BYTE* const base = ctx->base + ctx->dictLimit;
size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
const U32 maxSearches = 1 << ctx->params.searchLog;
@@ -1547,7 +1548,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* init */
ZSTD_resetSeqStore(seqStorePtr);
if (((ip-ctx->base) - ctx->dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
if ((ip-base) < REPCODE_STARTVALUE) ip = base + REPCODE_STARTVALUE;
/* Match Loop */
while (ip < ilimit)
@@ -1633,7 +1634,7 @@ size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* catch up */
if (offset)
{
while ((start>anchor) && (start>ctx->base+offset) && (start[-1] == start[-1-offset])) /* only search for offset within prefix */
while ((start>anchor) && (start>base+offset) && (start[-1] == start[-1-offset])) /* only search for offset within prefix */
{ start--; matchLength++; }
offset_2 = offset_1; offset_1 = offset;
}
@@ -1990,7 +1991,6 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
void* dst, size_t dstSize,
const void* src, size_t srcSize)
{
U32 adressOverflow = 0;
const BYTE* const ip = (const BYTE*) src;
/* Check if blocks follow each other */
@@ -2001,13 +2001,13 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
zc->lowLimit = zc->dictLimit;
zc->dictLimit = (U32)(zc->nextSrc - zc->base);
zc->dictBase = zc->base;
if ((size_t)zc->base < delta) adressOverflow = zc->lowLimit;
zc->base -= delta;
zc->nextToUpdate = zc->dictLimit;
if (zc->dictLimit - zc->lowLimit < 8) zc->lowLimit = zc->dictLimit; /* too small extDict */
}
/* preemptive overflow correction */
if (adressOverflow || (zc->lowLimit > (1<<30) ))
if ((zc->base > ip) || (zc->lowLimit > (1<<30) ))
{
U32 correction = zc->lowLimit-1;
ZSTD_reduceIndex(zc, correction);

2
programs/Makefile
View File

@@ -281,7 +281,7 @@ valgrindTest: zstd datagen fuzzer fullbench zbufftest
./datagen -g64MB > tmp
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
@rm tmp
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -T1mn -t1
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1
valgrind --leak-check=yes --error-exitcode=1 ./zbufftest -T1mn

41
programs/fuzzer.c
View File

@@ -261,6 +261,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
BYTE* srcBuffer;
BYTE* cBuffer;
BYTE* dstBuffer;
BYTE* mirrorBuffer;
size_t srcBufferSize = (size_t)1<<maxSrcLog;
size_t dstBufferSize = (size_t)1<<maxSampleLog;
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
@@ -268,18 +269,22 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
ZSTD_CCtx* ctx;
ZSTD_DCtx* dctx;
U32 startTime = FUZ_GetMilliStart();
/* allocation */
ctx = ZSTD_createCCtx();
dctx= ZSTD_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
mirrorBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !dstBuffer || !cBuffer || !ctx,
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|| !dstBuffer || !mirrorBuffer || !cBuffer || !ctx || !dctx,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
@@ -298,7 +303,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
{
size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
size_t cSize, dSize, dSupSize, errorCode;
size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
XXH64_state_t crc64;
U64 crcOrig, crcDest;
@@ -439,12 +444,13 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
}
}
/* Multi - segments compression test */
/* Streaming compression of scattered segments test */
XXH64_reset(&crc64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
totalTestSize = 0;
cSize = ZSTD_compressBegin(ctx, cBuffer, cBufferSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
for (n=0; n<nbChunks; n++)
@@ -457,25 +463,49 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize))
/* avoid invalid dstBufferTooSmall */
break;
if (totalTestSize+sampleSize > maxTestSize) break;
errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
totalTestSize += sampleSize;
if (totalTestSize > maxTestSize) break;
}
errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
crcOrig = XXH64_digest(&crc64);
/* streaming decompression test */
errorCode = ZSTD_resetDCtx(dctx);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)
{
size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
totalGenSize += genSize;
totalCSize += inSize;
}
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig)
errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);
}
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
_cleanup:
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(dctx);
free(cNoiseBuffer[0]);
free(cNoiseBuffer[1]);
free(cNoiseBuffer[2]);
@@ -483,6 +513,7 @@ _cleanup:
free(cNoiseBuffer[4]);
free(cBuffer);
free(dstBuffer);
free(mirrorBuffer);
return result;
_output_error:

2
programs/zstdcli.c
View File

@@ -119,7 +119,7 @@ static int usage(const char* programName)
DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -1 : Fast compression (default) \n");
DISPLAY( " -9 : High compression \n");
DISPLAY( " -19 : High compression \n");
DISPLAY( " -d : decompression (default for %s extension)\n", ZSTD_EXTENSION);
//DISPLAY( " -z : force compression\n");
DISPLAY( " -f : overwrite output without prompting \n");