lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-05 19:15:58 +03:00
Code Activity

zbuff properly supports small dst buffers

Browse Source
This commit is contained in:
Yann Collet
2015-11-27 14:07:36 +01:00
parent 944042ab7d
commit 31d1806a77
4 changed files with 51 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
lib/zstd_buffered.c
View File

@@ -204,11 +204,12 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
void* cDst; void* cDst;
size_t cSize; size_t cSize;
size_t iSize = zbc->inBuffPos - zbc->inToCompress; size_t iSize = zbc->inBuffPos - zbc->inToCompress;
if ((size_t)(oend-op) > ZSTD_compressBound(iSize)) size_t oSize = oend-op;
if (oSize >= ZSTD_compressBound(iSize))
cDst = op; /* compress directly into output buffer (avoid flush stage) */ cDst = op; /* compress directly into output buffer (avoid flush stage) */
else else
cDst = zbc->outBuff; cDst = zbc->outBuff, oSize = zbc->outBuffSize;
cSize = ZSTD_compressContinue(zbc->zc, cDst, oend-op, zbc->inBuff + zbc->inToCompress, iSize); cSize = ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
/* prepare next block */ /* prepare next block */
zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize; zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;

62
programs/Makefile
View File

@@ -171,60 +171,60 @@ test-zstd: zstd datagen
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen | ./zstd -6 -v | ./zstd -d | md5sum > tmp2 ./datagen | ./zstd -6 -v | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
@./datagen -g257MB | md5sum > tmp1 @./datagen -g270000000 | md5sum > tmp1
./datagen -g257MB | ./zstd -v | ./zstd -d | md5sum > tmp2 ./datagen -g270000000 | ./zstd -v | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g257MB | ./zstd -v2 | ./zstd -d | md5sum > tmp2 ./datagen -g270000000 | ./zstd -v2 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g257MB | ./zstd -v3 | ./zstd -d | md5sum > tmp2 ./datagen -g270000000 | ./zstd -v3 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
@./datagen -g129MB -P60| md5sum > tmp1 @./datagen -g140000000 -P60| md5sum > tmp1
./datagen -g129MB -P60 | ./zstd -v4 | ./zstd -d | md5sum > tmp2 ./datagen -g140000000 -P60 | ./zstd -v4 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g129MB -P60 | ./zstd -v5 | ./zstd -d | md5sum > tmp2 ./datagen -g140000000 -P60 | ./zstd -v5 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g129MB -P60 | ./zstd -v6 | ./zstd -d | md5sum > tmp2 ./datagen -g140000000 -P60 | ./zstd -v6 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
@./datagen -g65MB -P70 | md5sum > tmp1 @./datagen -g70000000 -P70 | md5sum > tmp1
./datagen -g65MB -P70 | ./zstd -v7 | ./zstd -d | md5sum > tmp2 ./datagen -g70000000 -P70 | ./zstd -v7 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g65MB -P70 | ./zstd -v8 | ./zstd -d | md5sum > tmp2 ./datagen -g70000000 -P70 | ./zstd -v8 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g65MB -P70 | ./zstd -v9 | ./zstd -d | md5sum > tmp2 ./datagen -g70000000 -P70 | ./zstd -v9 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
@./datagen -g33MB -P75 | md5sum > tmp1 @./datagen -g35000000 -P75 | md5sum > tmp1
./datagen -g33MB -P75 | ./zstd -v10 | ./zstd -d | md5sum > tmp2 ./datagen -g35000000 -P75 | ./zstd -v10 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g33MB -P75 | ./zstd -v11 | ./zstd -d | md5sum > tmp2 ./datagen -g35000000 -P75 | ./zstd -v11 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g33MB -P75 | ./zstd -v12 | ./zstd -d | md5sum > tmp2 ./datagen -g35000000 -P75 | ./zstd -v12 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
@./datagen -g17MB -P80 | md5sum > tmp1 @./datagen -g18000000 -P80 | md5sum > tmp1
./datagen -g17MB -P80 | ./zstd -v13 | ./zstd -d | md5sum > tmp2 ./datagen -g18000000 -P80 | ./zstd -v13 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g17MB -P80 | ./zstd -v14 | ./zstd -d | md5sum > tmp2 ./datagen -g18000000 -P80 | ./zstd -v14 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g17MB -P80 | ./zstd -v15 | ./zstd -d | md5sum > tmp2 ./datagen -g18000000 -P80 | ./zstd -v15 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g17MB -P80 | ./zstd -v16 | ./zstd -d | md5sum > tmp2 ./datagen -g18000000 -P80 | ./zstd -v16 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g17MB -P80 | ./zstd -v17 | ./zstd -d | md5sum > tmp2 ./datagen -g18000000 -P80 | ./zstd -v17 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
@./datagen -g49MB -P93 | md5sum > tmp1 @./datagen -g50000000 -P94 | md5sum > tmp1
./datagen -g49MB -P93 | ./zstd -v18 | ./zstd -d | md5sum > tmp2 ./datagen -g50000000 -P94 | ./zstd -v18 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g49MB -P93 | ./zstd -v19 | ./zstd -d | md5sum > tmp2 ./datagen -g50000000 -P94 | ./zstd -v19 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
@./datagen -g97MB -P99 | md5sum > tmp1 @./datagen -g99000000 -P99 | md5sum > tmp1
./datagen -g97MB -P99 | ./zstd -v20 | ./zstd -d | md5sum > tmp2 ./datagen -g99000000 -P99 | ./zstd -v20 | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
./datagen -g6GB -P99 | md5sum > tmp1 ./datagen -g6000000000 -P99| md5sum > tmp1
./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d | md5sum > tmp2 ./datagen -g6000000000 -P99| ./zstd -vq | ./zstd -d | md5sum > tmp2
@diff tmp1 tmp2 @diff tmp1 tmp2
test-zstd32: zstd32 datagen test-zstd32: zstd32 datagen
./datagen | ./zstd32 -v | ./zstd32 -d > $(VOID) ./datagen | ./zstd32 -v | ./zstd32 -d > $(VOID)
./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID)
./datagen -g6GB -P99 | ./zstd32 -vq | ./zstd32 -d > $(VOID) ./datagen -g6GB -P99 | ./zstd32 -vq | ./zstd32 -d > $(VOID)
test-fullbench: fullbench datagen test-fullbench: fullbench datagen

2
programs/fileio.c
View File

@@ -337,7 +337,6 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
size_t inStart=0; size_t inStart=0;
size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize); size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
if (toRead==0) break; /* end of Frame */
readSize -= inSize; readSize -= inSize;
inStart += inSize; inStart += inSize;
@@ -347,6 +346,7 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
frameSize += decodedSize; frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break;
if (readSize) continue; /* still some data left within inBuff */ if (readSize) continue; /* still some data left within inBuff */
/* Fill input buffer */ /* Fill input buffer */

23
programs/zbufftest.c
View File

@@ -324,22 +324,29 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
sampleSize = (size_t)1 << sampleSizeLog; sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, sampleSize);
readSize = sampleSize; readSize = sampleSize;
genSize = cBufferSize - cSize;
/* random size output buffer */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN (cBufferSize - cSize, sampleSize);
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize); errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
CHECK (readSize != sampleSize, "compression test condition not respected : input should be fully consumed")
XXH64_update(&crc64, srcBuffer+sampleStart, readSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readSize);
cSize += genSize; cSize += genSize;
totalTestSize += sampleSize; totalTestSize += readSize;
if ((FUZ_rand(&lseed) & 15) == 0) if ((FUZ_rand(&lseed) & 15) == 0)
{ {
/* add a few random flushes operations, to mess around */ /* add a few random flushes operations, to mess around */
genSize = cBufferSize - cSize; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN (cBufferSize - cSize, sampleSize);
errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize); errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize);
CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode)); CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode));
cSize += genSize; cSize += genSize;