lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

modified streaming decompression API

Browse Source
This commit is contained in:
Yann Collet
2016-01-26 15:58:49 +01:00
parent b923f65076
commit 7b51a2948f
9 changed files with 193 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
programs/bench.c
View File

@ -227,6 +227,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
void* const resultBuffer = malloc(srcSize);
ZSTD_CCtx* refCtx = ZSTD_createCCtx();
ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_DCtx* refDCtx = ZSTD_createDCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
U64 crcOrig = XXH64(srcBuffer, srcSize, 0);
U32 nbBlocks = 0;
@ -235,7 +236,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
/* Memory allocation & restrictions */
if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !dctx)
if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx)
EXM_THROW(31, "not enough memory");
/* Init blockTable data */
@ -298,7 +299,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_compress_insertDictionary(refCtx, dictBuffer, dictBufferSize);
for (blockNb=0; blockNb<nbBlocks; blockNb++)
{
ZSTD_duplicateCCtx(ctx, refCtx);
ZSTD_copyCCtx(ctx, refCtx);
size_t rSize = ZSTD_compressContinue(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
@ -323,41 +324,42 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
#if 1
/* Decompression */
memset(resultBuffer, 0xD6, srcSize);
memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
nbLoops = 0;
milliTime = BMK_GetMilliStart();
while (BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart();
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++) {
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
blockTable[blockNb].resSize = ZSTD_decompress_usingDict(dctx,
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
dictBuffer, dictBufferSize);
if (ZSTD_isError(blockTable[blockNb].resSize))
EXM_THROW(3, "ZSTD_decompress_usingDict() failed : %s", ZSTD_getErrorName(blockTable[blockNb].resSize));
} }
milliTime = BMK_GetMilliSpan(milliTime);
size_t regenSize;
regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
if (ZSTD_isError(regenSize))
EXM_THROW(3, "ZSTD_decompress_usingDict() failed : %s", ZSTD_getErrorName(regenSize));
blockTable[blockNb].resSize = regenSize;
} }
milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
/* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck)
{
if (crcOrig!=crcCheck) {
size_t u;
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++)
{
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u])
{
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
U32 segNb, bNb, pos;
size_t bacc = 0;
printf("Decoding error at pos %u ", (U32)u);
for (segNb = 0; segNb < nbBlocks; segNb++)
{
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + blockTable[segNb].srcSize > u) break;
bacc += blockTable[segNb].srcSize;
}
@ -365,8 +367,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
bNb = pos / (128 KB);
printf("(segment %u, block %u, pos %u) \n", segNb, bNb, pos);
break;
}
}
} }
break;
}
#endif
@ -375,7 +376,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (crcOrig == crcCheck)
DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
else
DISPLAY("X \n");
DISPLAY("\n");
}
/* clean up */
@ -383,6 +384,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
free(resultBuffer);
ZSTD_freeCCtx(refCtx);
ZSTD_freeCCtx(ctx);
ZSTD_freeDCtx(refDCtx);
ZSTD_freeDCtx(dctx);
return 0;
}

3
programs/fileio.c
View File

@ -529,8 +529,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
size_t readSize=alreadyLoaded;
/* Main decompression Loop */
ZBUFF_decompressInit(ress.dctx);
ZBUFF_decompressWithDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1)
{
/* Decode */

12
programs/fuzzer.c
View File

@ -207,7 +207,7 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig);
result = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
if (ZSTD_isError(result)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
@ -284,7 +284,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
result = ZSTD_decompressBegin(dctx);
if (ZSTD_isError(result)) goto _output_error;
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
@ -302,9 +302,8 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
result = ZSTD_resetDCtx(dctx);
result = ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize);
if (ZSTD_isError(result)) goto _output_error;
ZSTD_decompress_insertDictionary(dctx, CNBuffer, dictSize);
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result != blockSize) goto _output_error;
@ -574,7 +573,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_duplicateCCtx(ctx, refCtx);
errorCode = ZSTD_copyCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode));
totalTestSize = 0; cSize = 0;
for (n=0; n<nbChunks; n++)
@ -603,9 +602,8 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
crcOrig = XXH64_digest(xxh64);
/* streaming decompression test */
errorCode = ZSTD_resetDCtx(dctx);
errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)

6
programs/zbufftest.c
View File

@ -174,8 +174,7 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_decompressInit(zd);
ZBUFF_decompressWithDictionary(zd, CNBuffer, 128 KB);
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
readSize = cSize;
genSize = CNBufferSize;
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
@ -374,8 +373,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
crcOrig = XXH64_digest(xxh64);
/* multi - fragments decompression test */
ZBUFF_decompressInit(zd);
ZBUFF_decompressWithDictionary(zd, dict, dictSize);
ZBUFF_decompressInitDictionary(zd, dict, dictSize);
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize)