diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 2eea5feee..b2fcd322d 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -252,6 +252,6 @@ typedef struct { const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); - +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx); #endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h index e0f50cf40..126d6cf5c 100644 --- a/lib/common/zstd_static.h +++ b/lib/common/zstd_static.h @@ -51,7 +51,8 @@ extern "C" { /*-************************************* * Constants ***************************************/ -#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U /*-************************************* @@ -196,6 +197,7 @@ typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams; #define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ static const size_t ZSTD_frameHeaderSize_min = 5; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); diff --git a/lib/decompress/zbuff_decompress.c b/lib/decompress/zbuff_decompress.c index d4ca550a7..e785e9ec3 100644 --- a/lib/decompress/zbuff_decompress.c +++ b/lib/decompress/zbuff_decompress.c @@ -218,12 +218,19 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, break; } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; ip += neededInSize; - if (!decodedSize) break; /* this was just a header */ + if (!decodedSize) { + if (isSkipFrame) { + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = 0; + } + break; /* this was just a header */ + } zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; break; @@ -243,12 +250,21 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ - { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, zbd->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; zbd->inPos = 0; /* input is consumed */ - if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ + if (!decodedSize) { + if (isSkipFrame) { + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = 0; + break; + } + zbd->stage = ZBUFFds_read; /* this was just a header */ + break; + } zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; // break; /* ZBUFFds_flush follows */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b670c548c..64be8fb56 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -101,7 +101,8 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } * Context management ***************************************************************/ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage; + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; struct ZSTD_DCtx_s { @@ -312,7 +313,15 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t const BYTE* ip = (const BYTE*)src; if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; - if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ + fparamsPtr->frameContentSize = 0; + fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; + return 0; + } + return ERROR(prefix_unknown); + } /* ensure there is enough `srcSize` to fully read/decode frame header */ { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); @@ -995,6 +1004,11 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) return dctx->expected; } +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) +{ + return dctx->stage == ZSTDds_skipFrame; +} + size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { /* Sanity check */ @@ -1006,6 +1020,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { case ZSTDds_getFrameHeaderSize : if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); @@ -1063,6 +1083,17 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c dctx->previousDstEnd = (char*)dst + rSize; return rSize; } + case ZSTDds_decodeSkippableHeader: + { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); + dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); + dctx->stage = ZSTDds_skipFrame; + return 0; + } + case ZSTDds_skipFrame: + { dctx->expected = ZSTD_frameHeaderSize_min; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } default: return ERROR(GENERIC); /* impossible */ } diff --git a/programs/fileio.c b/programs/fileio.c index eba90d011..b333ded24 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -689,7 +689,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) continue; } #endif - if (magic != ZSTD_MAGICNUMBER) { + if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) { if (g_overwrite) /* -df : pass-through mode */ return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize); else { diff --git a/programs/zbufftest.c b/programs/zbufftest.c index fce0ab275..8d32bbcc8 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -146,7 +146,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo int testResult = 0; size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; void* CNBuffer = malloc(CNBufferSize); - size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + size_t const skippableFrameSize = 11; + size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); @@ -162,15 +163,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + /* generate skippable frame */ + MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32(compressedBuffer+4, (U32)skippableFrameSize); + cSize = skippableFrameSize + 8; /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); readSize = CNBufferSize; genSize = compressedBufferSize; - result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); + result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize); if (ZBUFF_isError(result)) goto _output_error; if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ - cSize = genSize; + cSize += genSize; genSize = compressedBufferSize - cSize; result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); if (result != 0) goto _output_error; /* error, or some data not flushed */