diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index db9417b30..2aa71b268 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -76,7 +76,7 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict) { size_t cSize; void* const cBuff = loadFile_orDie(fname, &cSize); - unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize); + unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==0) { fprintf(stderr, "%s : original size unknown \n", fname); exit(6); diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index 62a881fcb..3a75e164c 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -63,7 +63,7 @@ static void decompress(const char* fname) { size_t cSize; void* const cBuff = loadFile_X(fname, &cSize); - unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize); + unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==0) { printf("%s : original size unknown. Use streaming decompression instead. \n", fname); exit(5); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 9c04503d2..00c0d937e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -306,6 +306,100 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t return 0; } +static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize); + +/** ZSTD_getFrameContentSize() : +* compatible with legacy mode +* @return : decompressed size of the single frame pointed to be `src` if known, otherwise +* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined +* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { + ZSTD_frameParams fParams; + if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR; + if (fParams.windowSize == 0) { + /* Either skippable or empty frame, size == 0 either way */ + return 0; + } else if (fParams.frameContentSize != 0) { + return fParams.frameContentSize; + } else { + return ZSTD_CONTENTSIZE_UNKNOWN; + } + } +} + +/** ZSTD_findDecompressedSize() : + * compatible with legacy mode + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + { + unsigned long long totalDstSize = 0; + while (srcSize >= ZSTD_frameHeaderSize_prefix) { + const U32 magicNumber = MEM_readLE32(src); + + if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t skippableSize; + if (srcSize < ZSTD_skippableHeaderSize) + return ERROR(srcSize_wrong); + skippableSize = MEM_readLE32((const BYTE *)src + 4) + + ZSTD_skippableHeaderSize; + if (srcSize < skippableSize) { + /* srcSize_wrong */ + return 0; + } + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + + /* check for overflow */ + if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; + totalDstSize += ret; + } + { + size_t frameSrcSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) + { + frameSrcSize = ZSTD_frameSrcSizeLegacy(src, srcSize); + } + else +#endif + { + frameSrcSize = ZSTD_frameSrcSize(src, srcSize); + } + if (ZSTD_isError(frameSrcSize)) { + return 0; + } + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } + + if (srcSize) { + /* srcSize_wrong */ + return 0; + } + + return totalDstSize; + } +} /** ZSTD_getDecompressedSize() : * compatible with legacy mode @@ -316,14 +410,8 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t - frame header not complete (`srcSize` too small) */ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) { -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize); -#endif - { ZSTD_frameParams fparams; - size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize); - if (frResult!=0) return 0; - return fparams.frameContentSize; - } + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + return ret >= ZSTD_CONTENTSIZE_ERROR ? 0 : ret; } @@ -1363,6 +1451,48 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len return length; } +static size_t ZSTD_frameSrcSize(const void *src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + ZSTD_frameParams fParams; + + size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); + if (ZSTD_isError(headerSize)) return headerSize; + + /* Frame Header */ + { + size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); + if (ZSTD_isError(ret)) return ret; + if (ret > 0) return ERROR(srcSize_wrong); + } + + ip += headerSize; + remainingSize -= headerSize; + + /* Loop on each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + + if (blockProperties.lastBlock) break; + } + + if (fParams.checksumFlag) { /* Frame content checksum verification */ + if (remainingSize < 4) return ERROR(srcSize_wrong); + ip += 4; + remainingSize -= 4; + } + + return ip - ipstart; +} /*! ZSTD_decompressFrame() : * `dctx` must be properly initialized */ diff --git a/lib/dll/libzstd.def b/lib/dll/libzstd.def index 0a3259e9e..51d0c1925 100644 --- a/lib/dll/libzstd.def +++ b/lib/dll/libzstd.def @@ -58,6 +58,8 @@ EXPORTS ZSTD_getBlockSizeMax ZSTD_getCParams ZSTD_getDecompressedSize + ZSTD_findDecompressedSize + ZSTD_getFrameContentSize ZSTD_getErrorName ZSTD_getFrameParams ZSTD_getParams diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index 2a9f36aa2..c0369ab4f 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -123,6 +123,30 @@ MEM_STATIC size_t ZSTD_decompressLegacy( } } +MEM_STATIC size_t ZSTD_frameSrcSizeLegacy(const void *src, + size_t compressedSize) +{ + U32 const version = ZSTD_isLegacy(src, compressedSize); + switch(version) + { + case 1 : + return ZSTDv01_frameSrcSize(src, compressedSize); + case 2 : + return ZSTDv02_frameSrcSize(src, compressedSize); + case 3 : + return ZSTDv03_frameSrcSize(src, compressedSize); + case 4 : + return ZSTDv04_frameSrcSize(src, compressedSize); + case 5 : + return ZSTDv05_frameSrcSize(src, compressedSize); + case 6 : + return ZSTDv06_frameSrcSize(src, compressedSize); + case 7 : + return ZSTDv07_frameSrcSize(src, compressedSize); + default : + return ERROR(prefix_unknown); + } +} MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) { diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index 6fd30c0ac..c9b676ad8 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1992,6 +1992,37 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } +size_t ZSTDv01_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + U32 magicNumber; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + magicNumber = ZSTD_readBE32(src); + if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; + + /* Loop on each block */ + while (1) + { + size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv01_isError(blockSize)) return blockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (blockSize > remainingSize) return ERROR(srcSize_wrong); + + if (blockSize == 0) break; /* bt_end */ + + ip += blockSize; + remainingSize -= blockSize; + } + + return ip - (const BYTE*)src; +} /******************************* * Streaming Decompression API diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h index 0f2323db9..9e5d553bf 100644 --- a/lib/legacy/zstd_v01.h +++ b/lib/legacy/zstd_v01.h @@ -34,6 +34,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize); +/** +ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv01_isError()) +*/ +size_t ZSTDv01_frameSrcSize(const void* src, size_t compressedSize); + /** ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error */ diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index b8a12ab5e..f3b107af3 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -3378,6 +3378,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } +static size_t ZSTD_frameSrcSize(const void *src, size_t srcSize) +{ + + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + U32 magicNumber; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + magicNumber = MEM_readLE32(src); + if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; + + /* Loop on each block */ + while (1) + { + size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /******************************* * Streaming Decompression API @@ -3492,6 +3524,11 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); } +size_t ZSTDv02_frameSrcSize(const void *src, size_t compressedSize) +{ + return ZSTD_frameSrcSize(src, compressedSize); +} + ZSTDv02_Dctx* ZSTDv02_createDCtx(void) { return (ZSTDv02_Dctx*)ZSTD_createDCtx(); diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h index a371bd181..45dc5f6ca 100644 --- a/lib/legacy/zstd_v02.h +++ b/lib/legacy/zstd_v02.h @@ -34,6 +34,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize); +/** +ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv02_isError()) +*/ +size_t ZSTDv02_frameSrcSize(const void* src, size_t compressedSize); + /** ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error */ diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 6459da3d4..2eba77ccc 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -3019,6 +3019,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } +static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + U32 magicNumber; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + magicNumber = MEM_readLE32(src); + if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; + + /* Loop on each block */ + while (1) + { + size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} + /******************************* * Streaming Decompression API @@ -3133,6 +3165,11 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); } +size_t ZSTDv03_frameSrcSize(const void* src, size_t srcSize) +{ + return ZSTD_frameSrcSize(src, srcSize); +} + ZSTDv03_Dctx* ZSTDv03_createDCtx(void) { return (ZSTDv03_Dctx*)ZSTD_createDCtx(); diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h index 8b89737b1..24dba1f59 100644 --- a/lib/legacy/zstd_v03.h +++ b/lib/legacy/zstd_v03.h @@ -35,6 +35,14 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize); /** +ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv03_isError()) +*/ +size_t ZSTDv03_frameSrcSize(const void* src, size_t compressedSize); + + /** ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error */ unsigned ZSTDv03_isError(size_t code); diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 723242c6c..7c900e1e0 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3326,6 +3326,35 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx, return op-ostart; } +static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min; + + /* Loop on each block */ + while (1) + { + size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /* ****************************** * Streaming Decompression API @@ -3753,6 +3782,10 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t #endif } +size_t ZSTDv04_frameSrcSize(const void* src, size_t srcSize) +{ + return ZSTD_frameSrcSize(src, srcSize); +} size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); } diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h index 370553b18..671ba6dc0 100644 --- a/lib/legacy/zstd_v04.h +++ b/lib/legacy/zstd_v04.h @@ -34,6 +34,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize); +/** +ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv04_isError()) +*/ +size_t ZSTDv04_frameSrcSize(const void* src, size_t compressedSize); + /** ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error */ diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index f13592420..85c0f0e73 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3583,6 +3583,35 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t #endif } +size_t ZSTDv05_frameSrcSize(const void *src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong); + if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) return ERROR(prefix_unknown); + ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min; + + /* Loop on each block */ + while (1) + { + size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv05_isError(cBlockSize)) return cBlockSize; + + ip += ZSTDv05_blockHeaderSize; + remainingSize -= ZSTDv05_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /* ****************************** * Streaming Decompression API diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h index da26d96cf..ef2d18d10 100644 --- a/lib/legacy/zstd_v05.h +++ b/lib/legacy/zstd_v05.h @@ -32,6 +32,13 @@ extern "C" { size_t ZSTDv05_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +/** +ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv05_isError()) +*/ +size_t ZSTDv05_frameSrcSize(const void* src, size_t compressedSize); /* ************************************* * Helper functions diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 8be4bc377..92c1a45c6 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3729,6 +3729,37 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t #endif } +size_t ZSTDv06_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + blockProperties_t blockProperties = { bt_compressed, 0 }; + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min); + if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize; + if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown); + if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong); + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv06_isError(cBlockSize)) return cBlockSize; + + ip += ZSTDv06_blockHeaderSize; + remainingSize -= ZSTDv06_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /*_****************************** * Streaming Decompression API diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h index 14040abdd..1fad7311e 100644 --- a/lib/legacy/zstd_v06.h +++ b/lib/legacy/zstd_v06.h @@ -41,6 +41,13 @@ extern "C" { ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +/** +ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv06_isError()) +*/ +size_t ZSTDv06_frameSrcSize(const void* src, size_t compressedSize); /* ************************************* * Helper functions diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index b1fcdc766..4ee055ddd 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3968,6 +3968,38 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t #endif } +size_t ZSTDv07_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + + /* check */ + if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min); + if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize; + if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) return ERROR(prefix_unknown); + if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv07_isError(cBlockSize)) return cBlockSize; + + ip += ZSTDv07_blockHeaderSize; + remainingSize -= ZSTDv07_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /*_****************************** * Streaming Decompression API diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h index 30725dcf7..b02b3dce4 100644 --- a/lib/legacy/zstd_v07.h +++ b/lib/legacy/zstd_v07.h @@ -48,6 +48,14 @@ unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +/** +ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv07_isError()) +*/ +size_t ZSTDv07_frameSrcSize(const void* src, size_t compressedSize); + /*====== Helper functions ======*/ ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ diff --git a/lib/zstd.h b/lib/zstd.h index f5cbf4b48..90f0bca59 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -80,7 +80,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, int compressionLevel); /*! ZSTD_decompress() : - `compressedSize` : must be the _exact_ size of a single compressed frame. + `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. `dstCapacity` is an upper bound of originalSize. If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), @@ -88,7 +88,45 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) + +/*! ZSTD_getFrameContentSize() : +* `src` should point to the start of a ZSTD encoded frame +* @return : decompressed size of the frame pointed to be `src` if known, otherwise +* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined +* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */ +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_findDecompressedSize() : +* `src` should point the start of a series of ZSTD encoded and/or skippable frames +* `srcSize` must be the _exact_ size of this series +* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`) +* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_ +* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN +* - if an error occurred: ZSTD_CONTENTSIZE_ERROR +* +* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. +* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application can still use ZSTD_decompress() while relying on implied limits. +* (For example, data may be necessarily cut into blocks <= 16 KB). +* note 2 : decompressed size is always present when compression is done with ZSTD_compress() +* note 3 : decompressed size can be very large (64-bits value), +* potentially larger than what local system can handle as a single memory segment. +* In which case, it's necessary to use streaming mode to decompress data. +* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. +* Always ensure result fits within application's authorized limits. +* Each application can set its own limits. +* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to +* read each contained frame header. This is efficient as most of the data is skipped, +* however it does mean that all frame data must be present and valid. */ +ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + /*! ZSTD_getDecompressedSize() : +* WARNING: This function is now obsolete. ZSTD_findDecompressedSize should be used, +* or if only exactly one ZSTD frame is needed, ZSTD_getFrameContentSize can be used. +* * 'src' is the start of a zstd compressed frame. * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. diff --git a/programs/bench.c b/programs/bench.c index dcb23b1f2..73489643f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -184,7 +184,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, U64 dSize64 = 0; U32 fileNb; for (fileNb=0; fileNb