diff --git a/NEWS b/NEWS index 36aa45e91..35e0e5820 100644 --- a/NEWS +++ b/NEWS @@ -1,16 +1,19 @@ v1.1.2 -Improved : faster decompression speed at ultra compression settings and in 32-bits mode +API : streaming : decompression : changed : implicit reset on starting new frames without init +API : experimental : added : dictID retrieval functions +API : zbuff : changed : prototypes now generate deprecation warnings +lib : improved : faster decompression speed at ultra compression settings and 32-bits mode +lib : changed : only public ZSTD_ symbols are now exposed +lib : changed : reduced usage of stack memory +lib : fixed : several corner case bugs, by Nick Terrell cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski cli : new : preserve file attributes cli : new : added zstdless and zstdgrep tools cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) cli : fixed : zstdcat -API : changed : zbuff prototypes now generate deprecation warnings -API : changed : streaming decompression implicit reset on starting new frame -API : added experimental : dictID retrieval functions zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski -Changed : zbuff source files moved to lib/deprecated -Changed : reduced stack memory use +install : better compatibility with FreeBSD, by Dimitry Andric +source tree : changed : zbuff source files moved to lib/deprecated v1.1.1 New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp index 18c069eae..0b1403354 100644 --- a/contrib/pzstd/Options.cpp +++ b/contrib/pzstd/Options.cpp @@ -22,19 +22,13 @@ defined(__CYGWIN__) #include /* _isatty */ #define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || \ - defined(_POSIX_SOURCE) || \ - (defined(__APPLE__) && \ - defined( \ - __MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ \ - */ +#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \ + defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ #include /* isatty */ #define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else #define IS_CONSOLE(stdStream) 0 #endif -#endif namespace pzstd { diff --git a/examples/simple_compression.c b/examples/simple_compression.c index 332ce721b..deb0bbfc4 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -8,9 +8,9 @@ -#include // malloc, exit -#include // fprintf, perror -#include // strerror +#include // malloc, free, exit +#include // fprintf, perror, fopen, etc. +#include // strlen, strcat, memset, strerror #include // errno #include // stat #include // presumes zstd library is installed @@ -45,13 +45,18 @@ static void* malloc_orDie(size_t size) static void* loadFile_orDie(const char* fileName, size_t* size) { - off_t const buffSize = fsize_orDie(fileName); + off_t const fileSize = fsize_orDie(fileName); + size_t const buffSize = (size_t)fileSize; + if ((off_t)buffSize < fileSize) { /* narrowcast overflow */ + fprintf(stderr, "%s : filesize too large \n", fileName); + exit(4); + } FILE* const inFile = fopen_orDie(fileName, "rb"); void* const buffer = malloc_orDie(buffSize); size_t const readSize = fread(buffer, 1, buffSize, inFile); if (readSize != (size_t)buffSize) { fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); - exit(4); + exit(5); } fclose(inFile); /* can't fail, read only */ *size = buffSize; @@ -65,11 +70,11 @@ static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSi size_t const wSize = fwrite(buff, 1, buffSize, oFile); if (wSize != (size_t)buffSize) { fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno)); - exit(5); + exit(6); } if (fclose(oFile)) { perror(fileName); - exit(6); + exit(7); } } @@ -84,7 +89,7 @@ static void compress_orDie(const char* fname, const char* oname) size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1); if (ZSTD_isError(cSize)) { fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); - exit(7); + exit(8); } saveFile_orDie(oname, cBuff, cSize); diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index 108a63c83..4c2c1a1d8 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -7,11 +7,9 @@ */ -#include // malloc, exit -#include // fprintf, perror, feof -#include // strerror -#include // errno -#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being +#include // malloc, free, exit +#include // fprintf, perror, feof, fopen, etc. +#include // strlen, memset, strcat #include // presumes zstd library is installed @@ -82,7 +80,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } - if (toRead > buffInSize) toRead = buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */ + if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/ fwrite_orDie(buffOut, output.pos, fout); } } diff --git a/lib/Makefile b/lib/Makefile index 44c64b8ed..fcc0d099d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -47,9 +47,9 @@ ifeq ($(shell uname), Darwin) SHARED_EXT = dylib SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) - SONAME_FLAGS = -install_name $(PREFIX)/lib/$@.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) + SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) else - SONAME_FLAGS = -Wl,-soname=$@.$(SHARED_EXT).$(LIBVER_MAJOR) + SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR) SHARED_EXT = so SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) @@ -107,15 +107,15 @@ libzstd.pc: libzstd.pc.in install: libzstd.a libzstd libzstd.pc @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ - @install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) + @install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR) @cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR) @cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR) @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ - @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a - @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h - @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h - @install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # prototypes generate deprecation warnings - @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h + @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR) + @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR) + @install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR) + @install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR) # prototypes generate deprecation warnings + @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR) @echo zstd static and shared library installed uninstall: diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index a5002fb11..96e057758 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -147,7 +147,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } /*! ZSTD_wildcopy() : * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ #define WILDCOPY_OVERLENGTH 8 -MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length) +MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) { const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 665d09c0f..6e52ec8ec 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -33,6 +33,7 @@ typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZS /*-************************************* * Helper functions ***************************************/ +#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } @@ -148,6 +149,14 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) } +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; +} + /** ZSTD_adjustCParams() : optimize `cPar` for a given input (`srcSize` and `dictSize`). mostly downsizing to reduce memory consumption and initialization. @@ -166,9 +175,9 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; } } if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; - { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt) | (cPar.strategy == ZSTD_btopt2); - U32 const maxChainLog = cPar.windowLog+btPlus; - if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ + { U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog); + } if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ @@ -2274,16 +2283,16 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, if (remaining < blockSize) blockSize = remaining; /* preemptive overflow correction */ - if (cctx->lowLimit > (1<<30)) { - U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt) | (cctx->params.cParams.strategy == ZSTD_btopt2); - U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1; - U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */); - U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */ - U32 const correction = cctx->lowLimit - newLowLimit; + if (cctx->lowLimit > (2U<<30)) { + U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1; + U32 const current = (U32)(ip - cctx->base); + U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog); + U32 const correction = current - newCurrent; + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30); ZSTD_reduceIndex(cctx, correction); cctx->base += correction; cctx->dictBase += correction; - cctx->lowLimit = newLowLimit; + cctx->lowLimit -= correction; cctx->dictLimit -= correction; if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0; else cctx->nextToUpdate -= correction; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 70dd4ccaa..085477bf4 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -963,13 +963,13 @@ size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; - if (op > oend_w) { + if (op > oend_w || sequence.matchLength < MINMATCH) { U32 i; for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; return sequenceLength; } } } - /* Requirement: op <= oend_w */ + /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ /* match within prefix */ if (sequence.offset < 8) { @@ -997,7 +997,7 @@ size_t ZSTD_execSequence(BYTE* op, } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ } return sequenceLength; } @@ -1183,13 +1183,13 @@ size_t ZSTD_execSequenceLong(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; - if (op > oend_w) { + if (op > oend_w || sequence.matchLength < MINMATCH) { U32 i; for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; return sequenceLength; } } } - /* Requirement: op <= oend_w */ + /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ #endif /* match within prefix */ @@ -1218,7 +1218,7 @@ size_t ZSTD_execSequenceLong(BYTE* op, } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ } return sequenceLength; } diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index 5c36c2108..6fd30c0ac 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1354,7 +1354,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } -static void ZSTD_wildcopy(void* dst, const void* src, size_t length) +static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) { const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; @@ -1803,7 +1803,7 @@ static size_t ZSTD_execSequence(BYTE* op, } else { ZSTD_copy8(op, match); } op += 8; match += 8; - if (endMatch > oend-12) + if (endMatch > oend-(16-MINMATCH)) { if (op < oend-8) { @@ -1814,7 +1814,7 @@ static size_t ZSTD_execSequence(BYTE* op, while (op oend-12) + if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_8) { @@ -3218,7 +3218,7 @@ static size_t ZSTD_execSequence(BYTE* op, } else { - ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ } } diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 321450670..6459da3d4 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -2449,7 +2449,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */ -static void ZSTD_wildcopy(void* dst, const void* src, size_t length) +static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) { const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; @@ -2847,7 +2847,7 @@ static size_t ZSTD_execSequence(BYTE* op, } op += 8; match += 8; - if (oMatchEnd > oend-12) + if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_8) { @@ -2859,7 +2859,7 @@ static size_t ZSTD_execSequence(BYTE* op, } else { - ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ } } diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 11b5481a1..bd011319c 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -487,7 +487,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } /*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */ -static void ZSTD_wildcopy(void* dst, const void* src, size_t length) +static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) { const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; @@ -3107,7 +3107,7 @@ static size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; - if (op > oend_8) { + if (op > oend_8 || sequence.matchLength < MINMATCH) { while (op < oMatchEnd) *op++ = *match++; return sequenceLength; } @@ -3134,7 +3134,7 @@ static size_t ZSTD_execSequence(BYTE* op, } op += 8; match += 8; - if (oMatchEnd > oend-12) + if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_8) { @@ -3146,7 +3146,7 @@ static size_t ZSTD_execSequence(BYTE* op, } else { - ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ } return sequenceLength; } diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index bf1235a35..3dd740e5f 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -509,7 +509,7 @@ static void ZSTDv05_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } /*! ZSTDv05_wildcopy() : * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ -MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, size_t length) +MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, ptrdiff_t length) { const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; @@ -3325,7 +3325,7 @@ static size_t ZSTDv05_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; - if (op > oend_8) { + if (op > oend_8 || sequence.matchLength < MINMATCH) { while (op < oMatchEnd) *op++ = *match++; return sequenceLength; } @@ -3348,7 +3348,7 @@ static size_t ZSTDv05_execSequence(BYTE* op, } op += 8; match += 8; - if (oMatchEnd > oend-12) { + if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_8) { ZSTDv05_wildcopy(op, match, oend_8 - op); match += oend_8 - op; @@ -3357,7 +3357,7 @@ static size_t ZSTDv05_execSequence(BYTE* op, while (op < oMatchEnd) *op++ = *match++; } else { - ZSTDv05_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTDv05_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ } return sequenceLength; } diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 6584d4858..8be4bc377 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -537,7 +537,7 @@ static void ZSTDv06_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } /*! ZSTDv06_wildcopy() : * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ #define WILDCOPY_OVERLENGTH 8 -MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, size_t length) +MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, ptrdiff_t length) { const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; @@ -3470,7 +3470,7 @@ size_t ZSTDv06_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; - if (op > oend_8) { + if (op > oend_8 || sequence.matchLength < MINMATCH) { while (op < oMatchEnd) *op++ = *match++; return sequenceLength; } @@ -3503,7 +3503,7 @@ size_t ZSTDv06_execSequence(BYTE* op, } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTDv06_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTDv06_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ } return sequenceLength; } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 2ae6c5ad2..b607ec372 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -2845,7 +2845,7 @@ static void ZSTDv07_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } /*! ZSTDv07_wildcopy() : * custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ #define WILDCOPY_OVERLENGTH 8 -MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, size_t length) +MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, ptrdiff_t length) { const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; @@ -3693,7 +3693,7 @@ size_t ZSTDv07_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; - if (op > oend_w) { + if (op > oend_w || sequence.matchLength < MINMATCH) { while (op < oMatchEnd) *op++ = *match++; return sequenceLength; } @@ -3726,7 +3726,7 @@ size_t ZSTDv07_execSequence(BYTE* op, } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTDv07_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTDv07_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ } return sequenceLength; } diff --git a/programs/util.h b/programs/util.h index e7c6b0569..0fa70577a 100644 --- a/programs/util.h +++ b/programs/util.h @@ -25,7 +25,7 @@ extern "C" { # define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ - #define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ #endif #endif @@ -49,11 +49,11 @@ extern "C" { #include /* stat, utime */ #include /* stat */ #if defined(_MSC_VER) - #include /* utime */ - #include /* _chmod */ +# include /* utime */ +# include /* _chmod */ #else - #include /* chown, stat */ - #include /* utime */ +# include /* chown, stat */ +# include /* utime */ #endif #include /* time */ #include @@ -98,7 +98,7 @@ extern "C" { # define SET_HIGH_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) -# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L) +# if defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L)) # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } # else # define UTIL_sleepMilli(milli) /* disabled */ @@ -408,16 +408,15 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i { size_t pos; unsigned i, nbFiles; - char *bufend, *buf; + char* buf = (char*)malloc(LIST_SIZE_INCREASE); + char* bufend = buf + LIST_SIZE_INCREASE; const char** fileTable; - buf = (char*)malloc(LIST_SIZE_INCREASE); if (!buf) return NULL; - bufend = buf + LIST_SIZE_INCREASE; for (i=0, pos=0, nbFiles=0; i= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; buf = (char*)UTIL_realloc(buf, newListSize); @@ -439,8 +438,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); if (!fileTable) { free(buf); return NULL; } - for (i=0, pos=0; i /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ +#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \ + defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else diff --git a/tests/.gitignore b/tests/.gitignore index c8f9ae79e..b558ac258 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -11,6 +11,7 @@ datagen paramgrill paramgrill32 roundTripCrash +longmatch # Tmp test directory zstdtest diff --git a/tests/Makefile b/tests/Makefile index fbee21448..6110465f6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -124,6 +124,9 @@ datagen : $(PRGDIR)/datagen.c datagencli.c roundTripCrash : $(ZSTD_FILES) roundTripCrash.c $(CC) $(FLAGS) $^ -o $@$(EXT) +longmatch : $(ZSTD_FILES) longmatch.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + namespaceTest: if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi $(RM) $@ @@ -140,7 +143,7 @@ clean: fullbench-lib$(EXT) fullbench-dll$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ zstreamtest$(EXT) zstreamtest32$(EXT) \ - datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) + datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) @echo Cleaning completed @@ -180,7 +183,7 @@ zstd-playTests: datagen file $(ZSTD) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zstream +test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 @@ -237,4 +240,7 @@ test-zstream: zstreamtest test-zstream32: zstreamtest32 $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) +test-longmatch: longmatch + $(QEMU_SYS) ./longmatch + endif diff --git a/tests/longmatch.c b/tests/longmatch.c new file mode 100644 index 000000000..61b81b359 --- /dev/null +++ b/tests/longmatch.c @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +int compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size) { + ZSTD_inBuffer in = { data, size, 0 }; + while (in.pos < in.size) { + ZSTD_outBuffer tmp = out; + const size_t rc = ZSTD_compressStream(ctx, &tmp, &in); + if (ZSTD_isError(rc)) { + return 1; + } + } + { + ZSTD_outBuffer tmp = out; + const size_t rc = ZSTD_flushStream(ctx, &tmp); + if (rc != 0) { return 1; } + } + return 0; +} + +int main(int argc, const char** argv) { + ZSTD_CStream *ctx; + ZSTD_parameters params; + size_t rc; + unsigned windowLog; + (void)argc; + (void)argv; + /* Create stream */ + ctx = ZSTD_createCStream(); + if (!ctx) { return 1; } + /* Set parameters */ + memset(¶ms, 0, sizeof(params)); + params.cParams.windowLog = 18; + params.cParams.chainLog = 13; + params.cParams.hashLog = 14; + params.cParams.searchLog = 1; + params.cParams.searchLength = 7; + params.cParams.targetLength = 16; + params.cParams.strategy = ZSTD_fast; + windowLog = params.cParams.windowLog; + /* Initialize stream */ + rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0); + if (ZSTD_isError(rc)) { return 2; } + { + U64 compressed = 0; + const U64 toCompress = ((U64)1) << 33; + const size_t size = 1 << windowLog; + size_t pos = 0; + char *srcBuffer = (char*) malloc(1 << windowLog); + char *dstBuffer = (char*) malloc(ZSTD_compressBound(1 << windowLog)); + ZSTD_outBuffer out = { dstBuffer, ZSTD_compressBound(1 << windowLog), 0 }; + const char match[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const size_t randomData = (1 << windowLog) - 2*sizeof(match); + size_t i; + printf("\n === Long Match Test === \n"); + printf("Creating random data to produce long matches \n"); + for (i = 0; i < sizeof(match); ++i) { + srcBuffer[i] = match[i]; + } + for (i = 0; i < randomData; ++i) { + srcBuffer[sizeof(match) + i] = (char)(rand() & 0xFF); + } + for (i = 0; i < sizeof(match); ++i) { + srcBuffer[sizeof(match) + randomData + i] = match[i]; + } + printf("Compressing, trying to generate a segfault \n"); + if (compress(ctx, out, srcBuffer, size)) { + return 1; + } + compressed += size; + while (compressed < toCompress) { + const size_t block = rand() % (size - pos + 1); + if (pos == size) { pos = 0; } + if (compress(ctx, out, srcBuffer + pos, block)) { + return 1; + } + pos += block; + compressed += block; + } + printf("Compression completed successfully (no error triggered)\n"); + free(srcBuffer); + free(dstBuffer); + } + return 0; +} diff --git a/tests/playTests.sh b/tests/playTests.sh index 6179265e7..abde72c80 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -31,8 +31,10 @@ case "$OS" in ;; esac -case "$OSTYPE" in - darwin*) MD5SUM="md5 -r" ;; +UNAME=$(uname) +case "$UNAME" in + Darwin) MD5SUM="md5 -r" ;; + FreeBSD) MD5SUM="gmd5sum" ;; *) MD5SUM="md5sum" ;; esac @@ -228,8 +230,8 @@ cp ../programs/*.h dirTestDict $MD5SUM dirTestDict/* > tmph1 $ZSTD -f --rm dirTestDict/* -D tmpDictC $ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default -case "$OSTYPE" in - darwin*) $ECHO "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 +case "$UNAME" in + Darwin) $ECHO "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 *) $MD5SUM -c tmph1 ;; esac rm -rf dirTestDict