Merge remote-tracking branch 'refs/remotes/origin/dev' into dev070

# Conflicts: # .travis.yml # Makefile # lib/common/zstd_static.h # programs/Makefile # projects/VS2008/zstd/zstd.vcproj # projects/VS2008/zstdlib/zstdlib.vcproj # projects/cmake/lib/CMakeLists.txt # projects/cmake/programs/CMakeLists.txt
2025-08-08 17:22:10 +03:00 · 2016-06-01 00:07:09 +02:00
parent 5d500b60e7 cf6edcd0a1
commit a6b942018d
25 changed files with 247 additions and 295 deletions
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -38,7 +38,7 @@ MANDIR  = $(PREFIX)/share/man/man1

 ZSTDDIR = ../lib

-CPPFLAGS= -I$(ZSTDDIR)/common -DXXH_NAMESPACE=ZSTD_
+CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
 CFLAGS ?= -O3  # -falign-loops=32   # not always beneficial
 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
@@ -208,7 +208,7 @@ endif
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 zstd-playTests: datagen
-	ZSTD=$(ZSTD) ./tests/playTests.sh $(ZSTDRTTEST)
+	ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)

 test: test-zstd test-fullbench test-fuzzer test-zbuff

--- a/programs/dibio.h
+++ b/programs/dibio.h
@@ -32,7 +32,7 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
-#include "../lib/dictBuilder/zdict_static.h"   /* ZDICT_params_t */
+#include "zdict_static.h"   /* ZDICT_params_t */


 /*-*************************************
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -689,7 +689,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
                continue;
            }
 #endif
-            if (magic !=  ZSTD_MAGICNUMBER) {
+            if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) {
                if (g_overwrite)   /* -df : pass-through mode */
                    return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
                else {
--- a/programs/tests/playTests.sh
+++ b/programs/tests/playTests.sh
@@ -15,22 +15,30 @@ roundTripTest() {

    rm -f tmp1 tmp2
    $ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
-    ./datagen $1 $p | md5sum > tmp1
-    ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d  | md5sum > tmp2
+    ./datagen $1 $p | $MD5SUM > tmp1
+    ./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d  | $MD5SUM > tmp2
    diff -q tmp1 tmp2
 }

 isWindows=false
 ECHO="echo"
-if [[ "$OS" == "Windows"* ]]; then
+case "$OS" in
+  Windows*) 
    isWindows=true
    ECHO="echo -e"
+    ;;
+esac
+
+MD5SUM="md5sum"
+if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+    MD5SUM="md5 -r"
 fi

-$ECHO "\nStarting playTests.sh isWindows=$isWindows"
+$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME"

 [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"

+file $ZSTD
 $ECHO "\n**** simple tests **** "

 ./datagen > tmp
@@ -120,8 +128,8 @@ rm tmpSparse*
 $ECHO "\n**** dictionary tests **** "

 ./datagen > tmpDict
-./datagen -g1M | md5sum > tmp1
-./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
+./datagen -g1M | $MD5SUM > tmp1
+./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
 diff -q tmp1 tmp2
 $ECHO "Create first dictionary"
 $ZSTD --train *.c -o tmpDict
--- a/programs/tests/playTests.bat
+++ b/programs/tests/playTests.bat
@@ -1,157 +0,0 @@
-@echo off
-ECHO Start playTests.bat
-ECHO ZSTD^: %ZSTD%
-ECHO DATAGEN^: %DATAGEN%
-if "%ZSTD%"=="" echo ZSTD variable must be defined! && exit /b 1
-if "%DATAGEN%"=="" echo DATAGEN variable must be defined! && exit /b 1
-
-SET ROUNDTRIPTEST=tests\roundTripTest.bat
-
-echo. && echo **** simple tests ****
-%DATAGEN% > tmp
-%ZSTD% -f tmp                             && REM trivial compression case, creates tmp.zst
-%ZSTD% -df tmp.zst                        && REM trivial decompression case (overwrites tmp)
-echo test : too large compression level (must fail)
-%ZSTD% -99 tmp && (echo too large compression level undetected && exit /b 1)
-echo test : compress to stdout
-%ZSTD% tmp -c > tmpCompressed                 
-%ZSTD% tmp --stdout > tmpCompressed       && REM long command format
-echo test : null-length file roundtrip
-echo. 2>tmpEmpty | cat tmpEmpty | %ZSTD% - --stdout | %ZSTD% -d --stdout || (echo wrong null-length file roundtrip && exit /b 1)
-echo test : decompress file with wrong suffix (must fail)
-%ZSTD% -d tmpCompressed && (echo wrong suffix error not detected! && exit /b 1)
-%ZSTD% -d tmpCompressed -c > tmpResult    && REM decompression using stdout   
-%ZSTD% --decompress tmpCompressed -c > tmpResult
-%ZSTD% --decompress tmpCompressed --stdout > tmpResult
-REM %ZSTD% -d    < tmp.zst > NUL              && REM combine decompression, stdin & stdout
-REM %ZSTD% -d  - < tmp.zst > NUL
-%ZSTD% -dc   < tmp.zst > NUL
-%ZSTD% -dc - < tmp.zst > NUL
-%ZSTD% -q tmp && (echo overwrite check failed! && exit /b 1)
-%ZSTD% -q -f tmp
-%ZSTD% -q --force tmp
-%ZSTD% -df tmp && (echo should have refused : wrong extension && exit /b 1)
-
-echo. && echo **** Pass-Through mode ****
-echo "Hello world !" | %ZSTD% -df
-echo "Hello world !" | %ZSTD% -dcf
-
-
-echo. && echo **** frame concatenation ****
-
-echo hello > hello.tmp
-echo world! > world.tmp
-cat hello.tmp world.tmp > helloworld.tmp
-%ZSTD% -c hello.tmp > hello.zstd
-%ZSTD% -c world.tmp > world.zstd
-cat hello.zstd world.zstd > helloworld.zstd
-%ZSTD% -dc helloworld.zstd > result.tmp
-cat result.tmp
-fc /b helloworld.tmp result.tmp
-rm *.tmp *.zstd
-
-echo frame concatenation tests completed
-
-
-REM echo. && echo **** flush write error test ****
-
-REM echo echo foo ^| %ZSTD% ^> v:\full
-REM echo foo | %ZSTD% > v:\full && (echo write error not detected! && exit /b 1)
-REM echo "echo foo | %ZSTD% | %ZSTD% -d > /dev/full"
-REM echo foo | %ZSTD% | %ZSTD% -d > /dev/full && (echo write error not detected! && exit /b 1)
-
-
-echo. && echo **** dictionary tests ****
-
-%DATAGEN% > tmpDict
-%DATAGEN% -g1M | md5sum > tmp1
-%DATAGEN% -g1M | %ZSTD% -D tmpDict | %ZSTD% -D tmpDict -dvq | md5sum > tmp2
-fc tmp1 tmp2
-%ZSTD% --train *.c *.h -o tmpDict
-%ZSTD% xxhash.c -D tmpDict -of tmp
-%ZSTD% -d tmp -D tmpDict -of result
-fc xxhash.c result
-
-
-echo. && echo **** multiple files tests ****
-
-%DATAGEN% -s1        > tmp1 2> NUL
-%DATAGEN% -s2 -g100K > tmp2 2> NUL
-%DATAGEN% -s3 -g1M   > tmp3 2> NUL
-%ZSTD% -f tmp*
-echo compress tmp* :
-ls -ls tmp*
-rm tmp1 tmp2 tmp3
-echo decompress tmp* :
-%ZSTD% -df *.zst
-ls -ls tmp*
-
-echo compress tmp* into stdout ^> tmpall :
-%ZSTD% -c tmp1 tmp2 tmp3 > tmpall
-ls -ls tmp*
-echo decompress tmpall* into stdout ^> tmpdec :
-cp tmpall tmpall2
-%ZSTD% -dc tmpall* > tmpdec
-ls -ls tmp*
-echo compress multiple files including a missing one (notHere) :
-%ZSTD% -f tmp1 notHere tmp2 && (echo missing file not detected! && exit /b 1)
-
-
-echo. && echo **** integrity tests ****
-echo test one file (tmp1.zst)
-%ZSTD% -t tmp1.zst
-%ZSTD% --test tmp1.zst
-echo test multiple files (*.zst)
-%ZSTD% -t *.zst
-echo test good and bad files (*)
-%ZSTD% -t * && (echo bad files not detected! && exit /b 1)
-
-
-echo. && echo **** zstd round-trip tests ****
-
-CALL %ROUNDTRIPTEST%
-CALL %ROUNDTRIPTEST% -g15K       && REM TableID==3
-CALL %ROUNDTRIPTEST% -g127K      && REM TableID==2
-CALL %ROUNDTRIPTEST% -g255K      && REM TableID==1
-CALL %ROUNDTRIPTEST% -g513K      && REM TableID==0
-CALL %ROUNDTRIPTEST% -g512K 6    && REM greedy, hash chain
-CALL %ROUNDTRIPTEST% -g512K 16   && REM btlazy2 
-CALL %ROUNDTRIPTEST% -g512K 19   && REM btopt
-
-rm tmp*
-echo Param = %1
-if NOT "%1"=="--test-large-data" (
-    echo skipping large data tests
-    exit /b 0
-)
-
-CALL %ROUNDTRIPTEST% -g270000000 1
-CALL %ROUNDTRIPTEST% -g270000000 2
-CALL %ROUNDTRIPTEST% -g270000000 3
-
-CALL %ROUNDTRIPTEST% -g140000000 -P60 4
-CALL %ROUNDTRIPTEST% -g140000000 -P60 5
-CALL %ROUNDTRIPTEST% -g140000000 -P60 6
-
-CALL %ROUNDTRIPTEST% -g70000000 -P70 7
-CALL %ROUNDTRIPTEST% -g70000000 -P70 8
-CALL %ROUNDTRIPTEST% -g70000000 -P70 9
-
-CALL %ROUNDTRIPTEST% -g35000000 -P75 10
-CALL %ROUNDTRIPTEST% -g35000000 -P75 11
-CALL %ROUNDTRIPTEST% -g35000000 -P75 12
-
-CALL %ROUNDTRIPTEST% -g18000000 -P80 13
-CALL %ROUNDTRIPTEST% -g18000000 -P80 14
-CALL %ROUNDTRIPTEST% -g18000000 -P80 15
-CALL %ROUNDTRIPTEST% -g18000000 -P80 16
-CALL %ROUNDTRIPTEST% -g18000000 -P80 17
-
-CALL %ROUNDTRIPTEST% -g50000000 -P94 18
-CALL %ROUNDTRIPTEST% -g50000000 -P94 19
-
-CALL %ROUNDTRIPTEST% -g99000000 -P99 20
-CALL %ROUNDTRIPTEST% -g6000000000 -P99 1
-
-rm tmp*
-exit /b 0
--- a/programs/tests/roundTripTest.bat
+++ b/programs/tests/roundTripTest.bat
@@ -1,8 +0,0 @@
-@echo off
-if [%3]==[] (SET C=%2 && SET P=) ELSE (SET C=%3 && SET P=%2) 
-rm -f tmp1 tmp2
-echo roundTripTest: datagen %1 %P% ^| %ZSTD% -v%C% ^| %ZSTD% -d
-%DATAGEN% %1 %P% | md5sum > tmp1
-%DATAGEN% %1 %P% | %ZSTD% -vq%C% | %ZSTD% -d  | md5sum > tmp2
-fc tmp1 tmp2
-EXIT /B %ERRORLEVEL%
--- a/programs/zbufftest.c
+++ b/programs/zbufftest.c
@@ -147,11 +147,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    int testResult = 0;
    size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
    void* CNBuffer = malloc(CNBufferSize);
-    size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
+    size_t const skippableFrameSize = 11;
+    size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
    void* compressedBuffer = malloc(compressedBufferSize);
    size_t const decodedBufferSize = CNBufferSize;
    void* decodedBuffer = malloc(decodedBufferSize);
-    size_t result, cSize, readSize, genSize;
+    size_t result, cSize, readSize, readSkipSize, genSize;
    U32 testNb=0;
    ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
    ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
@@ -163,15 +164,19 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    }
    RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);

+    /* generate skippable frame */
+    MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
+    MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
+    cSize = skippableFrameSize + 8;
    /* Basic compression test */
    DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
    ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
    readSize = CNBufferSize;
    genSize = compressedBufferSize;
-    result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
+    result = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
    if (ZBUFF_isError(result)) goto _output_error;
    if (readSize != CNBufferSize) goto _output_error;   /* entire input should be consumed */
-    cSize = genSize;
+    cSize += genSize;
    genSize = compressedBufferSize - cSize;
    result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
    if (result != 0) goto _output_error;   /* error, or some data not flushed */
@@ -181,12 +186,17 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    /* Basic decompression test */
    DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
    ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
-    readSize = cSize;
+    readSkipSize = cSize;
    genSize = CNBufferSize;
-    result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
+    result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
+    if (genSize != 0) goto _output_error;   /* skippable frame */
+    ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
+    readSize = cSize - readSkipSize;
+    genSize = CNBufferSize;
+    result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
    if (result != 0) goto _output_error;  /* should reach end of frame == 0; otherwise, some data left, or an error */
    if (genSize != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
-    if (readSize != cSize) goto _output_error;   /* should have read the entire frame */
+    if (readSize+readSkipSize != cSize) goto _output_error;   /* should have read the entire frame */
    DISPLAYLEVEL(4, "OK \n");

    /* check regenerated data is byte exact */
@@ -200,17 +210,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo

    /* Byte-by-byte decompression test */
    DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
-    ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
-    {   size_t r = 1, pIn=0, pOut=0;
-        while (r) {
-            size_t inS = 1;
-            size_t outS = 1;
-            r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
-            pIn += inS;
-            pOut += outS;
-        }
-        readSize = pIn;
-        genSize = pOut;
+    {   size_t r, pIn=0, pOut=0;
+        do 
+        {   ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
+            r = 1;
+            while (r) {
+                size_t inS = 1;
+                size_t outS = 1;
+                r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
+                pIn += inS;
+                pOut += outS;
+            }
+            readSize = pIn;
+            genSize = pOut;
+        } while (genSize==0);
    }
    if (genSize != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
    if (readSize != cSize) goto _output_error;   /* should have read the entire frame */
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -52,7 +52,7 @@
 #  include <io.h>       /* _isatty */
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
 #else
-#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE
+#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE)
 #  include <unistd.h>   /* isatty */
 #  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
 #else