lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-09-01 04:42:03 +03:00
Code Activity

zbuff supports byte-by-byte decompression scenarios

Browse Source
This commit is contained in:
Yann Collet
2016-05-10 14:14:19 +02:00
parent 918b934a26
commit bd39d54576
4 changed files with 81 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
lib/decompress/zbuff_decompress.c
View File

@@ -64,14 +64,14 @@
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . * just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/ * *******************************************************************************/
typedef enum { ZBUFFds_init, ZBUFFds_readHeader, typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
/* *** Resource management *** */ /* *** Resource management *** */
struct ZBUFF_DCtx_s { struct ZBUFF_DCtx_s {
ZSTD_DCtx* zd; ZSTD_DCtx* zd;
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
size_t blockSize; ZBUFF_dStage stage;
char* inBuff; char* inBuff;
size_t inBuffSize; size_t inBuffSize;
size_t inPos; size_t inPos;
@@ -79,7 +79,9 @@ struct ZBUFF_DCtx_s {
size_t outBuffSize; size_t outBuffSize;
size_t outStart; size_t outStart;
size_t outEnd; size_t outEnd;
ZBUFF_dStage stage; size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t lhSize;
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
@@ -108,8 +110,8 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
{ {
zbd->stage = ZBUFFds_readHeader; zbd->stage = ZBUFFds_loadHeader;
zbd->inPos = zbd->outStart = zbd->outEnd = 0; zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
} }
@@ -139,15 +141,29 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
case ZBUFFds_init : case ZBUFFds_init :
return ERROR(init_missing); return ERROR(init_missing);
case ZBUFFds_readHeader : case ZBUFFds_loadHeader :
/* read header from src */ { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
{ size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr); if (hSize != 0) {
if (ZSTD_isError(headerSize)) return headerSize; size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
if (headerSize) { if (ZSTD_isError(hSize)) return hSize;
/* not enough input to decode header : needs headerSize > *srcSizePtr */ if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
*dstCapacityPtr = 0; memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
*srcSizePtr = 0; zbd->lhSize += iend-ip; ip = iend; notDone = 0;
return headerSize; *dstCapacityPtr = 0;
return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
break;
} }
/* Consume header */
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */
size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
if (ZSTD_isError(h1Result)) return h1Result;
if (h1Size < zbd->lhSize) { /* long header */
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
if (ZSTD_isError(h2Result)) return h2Result;
} } } }
/* Frame header instruct buffer sizes */ /* Frame header instruct buffer sizes */
@@ -175,8 +191,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
notDone = 0; notDone = 0;
break; break;
} }
if ((size_t)(iend-ip) >= neededInSize) { if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
/* directly decode from src */
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
ip, neededInSize); ip, neededInSize);
@@ -200,6 +215,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
ip += loadedSize; ip += loadedSize;
zbd->inPos += loadedSize; zbd->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
/* decode loaded input */ /* decode loaded input */
{ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,

42
lib/decompress/zstd_decompress.c
View File

@@ -657,7 +657,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
} }
FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, //FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence, BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_8, const BYTE** litPtr, const BYTE* const litLimit_8,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
@@ -964,31 +965,29 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
return dctx->expected; return dctx->expected;
} }
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
/* Sanity check */ /* Sanity check */
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
ZSTD_checkContinuity(dctx, dst); if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
/* Decompress : frame header; part 1 */ /* Decompress : frame header; part 1 */
switch (dctx->stage) switch (dctx->stage)
{ {
case ZSTDds_getFrameHeaderSize : case ZSTDds_getFrameHeaderSize :
{ if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); if (dctx->headerSize > ZSTD_frameHeaderSize_min) {
if (dctx->headerSize > ZSTD_frameHeaderSize_min) { dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min;
dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; dctx->stage = ZSTDds_decodeFrameHeader;
dctx->stage = ZSTDds_decodeFrameHeader; return 0;
return 0;
}
dctx->expected = 0; /* not necessary to copy more */
} }
dctx->expected = 0; /* not necessary to copy more */
case ZSTDds_decodeFrameHeader: case ZSTDds_decodeFrameHeader:
{ { size_t result;
size_t result;
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
if (ZSTD_isError(result)) return result; if (ZSTD_isError(result)) return result;
@@ -997,8 +996,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
return 0; return 0;
} }
case ZSTDds_decodeBlockHeader: case ZSTDds_decodeBlockHeader:
{ { blockProperties_t bp;
blockProperties_t bp;
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(cBlockSize)) return cBlockSize; if (ZSTD_isError(cBlockSize)) return cBlockSize;
if (bp.blockType == bt_end) { if (bp.blockType == bt_end) {
@@ -1012,16 +1010,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
return 0; return 0;
} }
case ZSTDds_decompressBlock: case ZSTDds_decompressBlock:
{ { size_t rSize;
/* Decompress : block content */
size_t rSize;
switch(dctx->bType) switch(dctx->bType)
{ {
case bt_compressed: case bt_compressed:
rSize = ZSTD_decompressBlock_internal(dctx, dst, maxDstSize, src, srcSize); rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
break; break;
case bt_raw : case bt_raw :
rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize); rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
break; break;
case bt_rle : case bt_rle :
return ERROR(GENERIC); /* not yet handled */ return ERROR(GENERIC); /* not yet handled */

6
programs/fileio.c
View File

@@ -533,9 +533,9 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
/* Complete Header loading */ /* Header loading (optional, saves one loop) */
{ size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */ { size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded; /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); /* can be <= toLoad (null string) */ size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
readSize = alreadyLoaded + loadedSize; readSize = alreadyLoaded + loadedSize;
} }

27
programs/zbufftest.c
View File

@@ -188,6 +188,33 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
} }
/* Byte-by-byte decompression test */
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
{ size_t r = 1, pIn=0, pOut=0;
while (r) {
size_t inS = 1;
size_t outS = 1;
r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
pIn += inS;
pOut += outS;
}
readSize = pIn;
genSize = pOut;
}
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
{ size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
}
DISPLAYLEVEL(4, "OK \n");
}
_end: _end:
ZBUFF_freeCCtx(zc); ZBUFF_freeCCtx(zc);
ZBUFF_freeDCtx(zd); ZBUFF_freeDCtx(zd);