diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 889061496..34c0c4c04 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -93,6 +93,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code) case PREFIX(no_error): return "No error detected"; case PREFIX(GENERIC): return "Error (generic)"; case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(parameter_unknown): return "Unknown parameter type"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; diff --git a/lib/common/error_public.h b/lib/common/error_public.h index 29050b3b3..10f020cfe 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -37,6 +37,9 @@ extern "C" { #endif +/*===== dependency =====*/ +#include /* size_t */ + /* **************************************** * error codes list @@ -45,6 +48,7 @@ typedef enum { ZSTD_error_no_error, ZSTD_error_GENERIC, ZSTD_error_prefix_unknown, + ZSTD_error_parameter_unknown, ZSTD_error_frameParameter_unsupported, ZSTD_error_frameParameter_unsupportedBy32bits, ZSTD_error_compressionParameter_unsupported, diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a32cb5ad3..e0ae91bf8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1230,9 +1230,20 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, offset = (U32)(ip-matchLong); while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) { - mLength = ZSTD_count(ip+4, match+4, iend) + 4; - offset = (U32)(ip-match); - while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* match3 = base + matchIndex3; + hashLong[h3] = current + 1; + if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + mLength = ZSTD_count(ip+9, match3+8, iend) + 8; + ip++; + offset = (U32)(ip-match3); + while (((ip>anchor) & (match3>lowest)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip-match); + while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } } else { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; @@ -1359,16 +1370,32 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, offset_2 = offset_1; offset_1 = offset; ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) { - const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; - const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < dictLimit ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; U32 offset; - mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; - while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset = current - matchIndex; + hashLong[h3] = current + 1; + if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* matchEnd = matchIndex3 < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, lowPrefixPtr) + 8; + ip++; + offset = current+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; + offset = current - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } offset_2 = offset_1; offset_1 = offset; ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } else { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; @@ -3031,7 +3058,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - not used */ { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ - { 20, 16, 18, 1, 5, 16, ZSTD_dfast }, /* level 3 */ + { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */ { 20, 13, 17, 2, 5, 16, ZSTD_greedy }, /* level 4.*/ { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ @@ -3054,24 +3081,24 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, { /* for srcSize <= 256 KB */ /* W, C, H, S, L, T, strat */ - { 18, 12, 12, 1, 7, 4, ZSTD_fast }, /* level 0 - not used */ - { 18, 13, 14, 1, 6, 4, ZSTD_fast }, /* level 1 */ - { 18, 15, 17, 1, 5, 4, ZSTD_fast }, /* level 2 */ - { 18, 13, 15, 1, 5, 4, ZSTD_greedy }, /* level 3.*/ - { 18, 15, 17, 1, 5, 4, ZSTD_greedy }, /* level 4.*/ - { 18, 16, 17, 4, 5, 4, ZSTD_greedy }, /* level 5 */ - { 18, 17, 17, 5, 5, 4, ZSTD_greedy }, /* level 6 */ - { 18, 17, 17, 4, 4, 4, ZSTD_lazy }, /* level 7 */ - { 18, 17, 17, 4, 4, 4, ZSTD_lazy2 }, /* level 8 */ - { 18, 17, 17, 5, 4, 4, ZSTD_lazy2 }, /* level 9 */ - { 18, 17, 17, 6, 4, 4, ZSTD_lazy2 }, /* level 10 */ - { 18, 18, 17, 6, 4, 4, ZSTD_lazy2 }, /* level 11.*/ - { 18, 18, 17, 7, 4, 4, ZSTD_lazy2 }, /* level 12.*/ - { 18, 19, 17, 7, 4, 4, ZSTD_btlazy2 }, /* level 13 */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */ + { 18, 13, 14, 1, 6, 8, ZSTD_fast }, /* level 1 */ + { 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */ + { 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */ + { 18, 15, 17, 1, 5, 8, ZSTD_greedy }, /* level 4.*/ + { 18, 16, 17, 4, 5, 8, ZSTD_greedy }, /* level 5.*/ + { 18, 16, 17, 3, 5, 8, ZSTD_lazy }, /* level 6.*/ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy }, /* level 7 */ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 11.*/ + { 18, 18, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 12.*/ + { 18, 19, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13 */ { 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/ - { 18, 18, 18, 8, 4, 24, ZSTD_btopt }, /* level 15.*/ - { 18, 19, 18, 8, 3, 48, ZSTD_btopt }, /* level 16.*/ - { 18, 19, 18, 8, 3, 96, ZSTD_btopt }, /* level 17.*/ + { 18, 18, 18, 4, 3, 16, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/ + { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ { 18, 19, 18, 11, 3,512, ZSTD_btopt }, /* level 20.*/ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index c3d9d608c..b96f487ab 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -51,12 +51,12 @@ /*! -* STREAM_WINDOW_MAX : -* maximum window size accepted by DStream. -* frames requiring more memory will be rejected. +* MAXWINDOWSIZE_DEFAULT : +* maximum window size accepted by DStream, by default. +* Frames requiring more memory will be rejected. */ -#ifndef ZSTD_STREAM_WINDOW_MAX -# define ZSTD_STREAM_WINDOW_MAX (257 << 20) /* 257 MB */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (257 << 20) /* 257 MB */ #endif @@ -1328,6 +1328,7 @@ struct ZSTD_DStream_s { char* inBuff; size_t inBuffSize; size_t inPos; + size_t maxWindowSize; char* outBuff; size_t outBuffSize; size_t outStart; @@ -1361,6 +1362,7 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) zds->zd = ZSTD_createDCtx_advanced(customMem); if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } zds->stage = zdss_init; + zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; return zds; } @@ -1392,6 +1394,18 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } +size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, + ZSTD_DStreamParameter_e paramType, unsigned paramValue) +{ + switch(paramType) + { + default : return ERROR(parameter_unknown); + case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break; + } + return 0; +} + + size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; @@ -1450,11 +1464,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); /* Frame header instruct buffer sizes */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); size_t const neededOutSize = zds->fParams.windowSize + blockSize; - if (zds->fParams.windowSize > ZSTD_STREAM_WINDOW_MAX) return ERROR(frameParameter_unsupported); zds->blockSize = blockSize; if (zds->inBuffSize < blockSize) { zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); diff --git a/lib/zstd.h b/lib/zstd.h index cb667484b..fd6170c8a 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -87,7 +87,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, * When `return==0`, data to decompress can have any size. * In which case, it's necessary to use streaming mode to decompress data. * Optionally, application may rely on its own implied limits. -* (For example, application own data could be necessarily cut into blocks <= 16 KB). +* (For example, application data could be necessarily cut into blocks <= 16 KB). * note 3 : decompressed size could be wrong or intentionally modified ! * Always ensure result fits within application's authorized limits ! * Each application can set its own limits. @@ -264,7 +264,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * * A ZSTD_DStream object is required to track streaming operations. * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-init multiple times. +* ZSTD_DStream objects can be re-used multiple times. * * Use ZSTD_initDStream() to start a new decompression operation, * or ZSTD_initDStream_usingDict() if decompression requires a dictionary. @@ -422,7 +422,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); /* ****************************************************************** -* Advanced Streaming +* Advanced Streaming functions ********************************************************************/ /*====== compression ======*/ @@ -436,9 +436,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /*====== decompression ======*/ -/* advanced */ +typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; + ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 4c81e8df6..edfb07eb7 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -212,6 +212,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000000000); /* large limit */ + if (ZSTD_isError(r)) goto _output_error; } { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ @@ -267,6 +269,27 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo } } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++); + { size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1); /* large limit */ + if (!ZSTD_isError(r)) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + + /* Memory restriction */ + DISPLAYLEVEL(4, "test%3i : maxWindowSize < frame requirement : ", testNb++); + ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000); /* too small limit */ + if (ZSTD_isError(r)) goto _output_error; } + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */ + DISPLAYLEVEL(4, "OK (%s)\n", ZSTD_getErrorName(r)); } + + _end: ZSTD_freeCStream(zc); ZSTD_freeDStream(zd);