Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

.travis.yml

@@ -22,7 +22,7 @@ matrix:
           packages:
             - gcc-4.8
             - g++-4.8
-      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean"
     - os: linux
       sudo: false
       env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"

Makefile

@@ -20,28 +20,32 @@ else
 VOID = /dev/null
 endif
 
-.PHONY: default all zlibwrapper zstd clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
+.PHONY: default
-
+default: lib zstd
-default: libzstd zstd
 
+.PHONY: all
 all:
 	$(MAKE) -C $(ZSTDDIR) $@
 	$(MAKE) -C $(PRGDIR) $@ zstd32
 	$(MAKE) -C $(TESTDIR) $@ all32
 
-libzstd:
+.PHONY: lib
+lib:
 	@$(MAKE) -C $(ZSTDDIR)
 
 zstd:
 	@$(MAKE) -C $(PRGDIR)
 	cp $(PRGDIR)/zstd .
 
+.PHONY: zlibwrapper
 zlibwrapper:
 	$(MAKE) -C $(ZWRAPDIR) test
 
+.PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
 
+.PHONY: clean
 clean:
 	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@@ -56,6 +60,8 @@ clean:
 #------------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
 HOST_OS = POSIX
+.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
+
 install:
 	@$(MAKE) -C $(ZSTDDIR) $@
 	@$(MAKE) -C $(PRGDIR) $@

lib/Makefile

@@ -26,8 +26,9 @@ INCLUDEDIR=$(PREFIX)/include
 CPPFLAGS= -I. -I./common
 CFLAGS ?= -O3
 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-          -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
+          -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
+          -Wpointer-arith
+FLAGS   = $(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)
 
 
 ZSTD_FILES := common/*.c compress/*.c decompress/*.c dictBuilder/*.c
@@ -39,7 +40,6 @@ ZSTD_FILES+= legacy/*.c
 CPPFLAGS  += -I./legacy -DZSTD_LEGACY_SUPPORT=1
 endif
 
-
 # OS X linker doesn't support -soname, and use different extension
 # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
 ifeq ($(shell uname), Darwin)
@@ -54,22 +54,32 @@ else
 	SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
 endif
 
+LIBZSTD = libzstd.$(SHARED_EXT_VER)
+
 
 .PHONY: default all clean install uninstall
 
-default: clean libzstd
+default: lib
 
-all: clean libzstd
+all: lib
 
-libzstd: $(ZSTD_FILES)
+libzstd.a: ARFLAGS = rcs
+libzstd.a: $(ZSTD_FILES)
 	@echo compiling static library
 	@$(CC) $(FLAGS) -c $^
-	@$(AR) rcs $@.a *.o
+	@$(AR) $(ARFLAGS) $@ *.o
+
+$(LIBZSTD): LDFLAGS += -shared -fPIC
+$(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
-	@$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
+	@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
-	@ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR)
+	@ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT_MAJOR)
-	@ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT)
+	@ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT)
+
+libzstd : $(LIBZSTD)
+
+lib: libzstd.a libzstd
 
 clean:
 	@rm -f core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
@@ -89,7 +99,7 @@ libzstd.pc: libzstd.pc.in
              -e 's|@VERSION@|$(VERSION)|' \
              $< >$@
 
-install: libzstd libzstd.pc
+install: libzstd.a libzstd libzstd.pc
 	@install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
 	@install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
 	@cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)

lib/decompress/zstd_decompress.c

@@ -193,7 +193,24 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 *   Decompression section
 ***************************************************************/
 
-/* See compression format details in : doc/zstd_compression_format.md */
+/*! ZSTD_isFrame() :
+ *  Tells if the content of `buffer` starts with a valid Frame Identifier.
+ *  Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ *  Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ *  Note 3 : Skippable Frame Identifiers are considered valid. */
+unsigned ZSTD_isFrame(const void* buffer, size_t size)
+{
+    if (size < 4) return 0;
+    {   U32 const magic = MEM_readLE32(buffer);
+        if (magic == ZSTD_MAGICNUMBER) return 1;
+        if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
+    }
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+    if (ZSTD_isLegacy(buffer, size)) return 1;
+#endif
+    return 0;
+}
+
 
 /** ZSTD_frameHeaderSize() :
 *   srcSize must be >= ZSTD_frameHeaderSize_prefix.

lib/dictBuilder/zdict.c

@@ -898,12 +898,14 @@ size_t ZDICT_trainFromBuffer_unsafe(
         U32 const nb = MIN(25, dictList[0].pos);
         U32 const dictContentSize = ZDICT_dictSize(dictList);
         U32 u;
-        DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize);
+        DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos-1, dictContentSize);
-        DISPLAYLEVEL(3, "list %u best segments \n", nb);
+        DISPLAYLEVEL(3, "list %u best segments \n", nb-1);
-        for (u=1; u<=nb; u++) {
+        for (u=1; u<nb; u++) {
-            U32 pos = dictList[u].pos;
+            U32 const pos = dictList[u].pos;
-            U32 length = dictList[u].length;
+            U32 const length = dictList[u].length;
-            U32 printedLength = MIN(40, length);
+            U32 const printedLength = MIN(40, length);
+            if ((pos > samplesBuffSize) || ((pos + length) > samplesBuffSize))
+                return ERROR(GENERIC);   /* should never happen */
             DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
                          u, length, pos, dictList[u].savings);
             ZDICT_printHex((const char*)samplesBuffer+pos, printedLength);

lib/zstd.h

@@ -433,6 +433,13 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
 
 /*--- Advanced decompression functions ---*/
 
+/*! ZSTD_isFrame() :
+ *  Tells if the content of `buffer` starts with a valid Frame Identifier.
+ *  Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+ *  Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+ *  Note 3 : Skippable Frame Identifiers are considered valid. */
+ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
+
 /*! ZSTD_estimateDCtxSize() :
  *  Gives the potential amount of memory allocated to create a ZSTD_DCtx */
 ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);

programs/Makefile

@@ -30,7 +30,8 @@ endif
 CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder
 CFLAGS ?= -O3
 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-          -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
+          -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
+          -Wpointer-arith
 CFLAGS += $(MOREFLAGS)
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
@@ -41,7 +42,6 @@ ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
 ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
-ZSTDDECOMP32_O = $(ZSTDDIR)/decompress/zstd_decompress32.o
 
 ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
 CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0
@@ -76,30 +76,27 @@ default: zstd
 
 all: zstd
 
-$(ZSTDDECOMP_O): CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
-$(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
 
+zstd  : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 zstd  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
         zstdcli.c fileio.c bench.c datagen.c dibio.c
 ifneq (,$(filter Windows%,$(OS)))
 	windres\generate_res.bat
 endif
-	$(CC)      $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT)
+	$(CC)      $(FLAGS) $^ $(RES_FILE) -o $@$(EXT)
 
 
-$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
+zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-	$(CC)  -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
+zstd32 : $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
-
-zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
         zstdcli.c fileio.c bench.c datagen.c dibio.c
 ifneq (,$(filter Windows%,$(OS)))
 	windres\generate_res.bat
 endif
-	$(CC)  -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT)
+	$(CC)  -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
 
 
-zstd_nolegacy : clean_decomp_o
+zstd-nolegacy : clean_decomp_o
 	$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
 
 zstd-pgo : MOREFLAGS = -fprofile-generate
@@ -111,7 +108,7 @@ zstd-pgo : clean zstd
 	./zstd -b7i2 $(PROFILE_WITH)
 	./zstd -b5 $(PROFILE_WITH)
 	$(RM) zstd
-	$(RM) $(ZSTDDIR)/decompress/zstd_decompress.o
+	$(RM) $(ZSTDDECOMP_O)
 	$(MAKE) zstd MOREFLAGS=-fprofile-use
 
 zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c
@@ -142,7 +139,6 @@ clean:
 
 clean_decomp_o:
 	@$(RM) $(ZSTDDECOMP_O)
-	@$(RM) $(ZSTDDECOMP32_O)
 
 
 #----------------------------------------------------------------------------------

programs/fileio.c

@@ -8,16 +8,6 @@
  */
 
 
-/* *************************************
- *  Tuning options
- ***************************************/
-#ifndef ZSTD_LEGACY_SUPPORT
-/* LEGACY_SUPPORT :
- *  decompressor can decode older formats (starting from zstd 0.1+) */
-#  define ZSTD_LEGACY_SUPPORT 1
-#endif
-
-
 /* *************************************
 *  Compiler Options
 ***************************************/
@@ -29,6 +19,7 @@
 #  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */
 #endif
 
+
 /*-*************************************
 *  Includes
 ***************************************/
@@ -44,10 +35,6 @@
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
 #include "zstd.h"
 
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-#  include "zstd_legacy.h"    /* ZSTD_isLegacy */
-#endif
-
 
 /*-*************************************
 *  OS-specific Includes
@@ -638,7 +625,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
     @return : 0 (no error) */
 static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize)
 {
-    size_t const blockSize = MIN (64 KB, bufferSize);
+    size_t const blockSize = MIN(64 KB, bufferSize);
     size_t readFromInput = 1;
     unsigned storedSkips = 0;
 
@@ -686,21 +673,16 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
         }
         readSomething = 1;   /* there is at least >= 4 bytes in srcFile */
         if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
-        {   U32 const magic = MEM_readLE32(ress.srcBuffer);
+        if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
-            if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) & (magic != ZSTD_MAGICNUMBER)
+            if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) {  /* pass-through mode */
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+                unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
-                  & (!ZSTD_isLegacy(ress.srcBuffer, toRead))
+                if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName);  /* error should never happen */
-#endif
+                return result;
-                ) {
+            } else {
-                if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) {  /* pass-through mode */
+                DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
-                    unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
+                fclose(srcFile);
-                    if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName);  /* error should never happen */
+                return 1;
-                    return result;
+        }   }
-                } else {
-                    DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
-                    fclose(srcFile);
-                    return 1;
-        }   }   }
         filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead);
     }
 

tests/Makefile

@@ -76,7 +76,7 @@ zstd:
 zstd32:
 	$(MAKE) -C $(PRGDIR) $@
 
-zstd_nolegacy:
+zstd-nolegacy:
 	$(MAKE) -C $(PRGDIR) $@
 
 fullbench  : $(ZSTD_FILES) $(PRGDIR)/datagen.c fullbench.c
@@ -181,8 +181,8 @@ test-zstd: zstd zstd-playTests
 test-zstd32: ZSTD = $(PRGDIR)/zstd32
 test-zstd32: zstd32 zstd-playTests
 
-test-zstd_nolegacy: ZSTD = $(PRGDIR)/zstd
+test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd
-test-zstd_nolegacy: zstd_nolegacy zstd-playTests
+test-zstd-nolegacy: zstd-nolegacy zstd-playTests
 
 test-fullbench: fullbench datagen
 	./fullbench -i1

zlibWrapper/.gitignore

@@ -3,10 +3,16 @@ _*
 example.*
 example_zstd.*
 fitblk.*
-fitblk_zstd.* 
+fitblk_zstd.*
 zwrapbench.*
 foo.gz
 
+example
+example_zstd
+fitblk
+fitblk_zstd
+zwrapbench
+
 # Misc files
 *.bat
 *.zip

zlibWrapper/Makefile

@@ -16,7 +16,7 @@ EXAMPLE_PATH = examples
 PROGRAMS_PATH = ../programs
 TEST_FILE = ../doc/zstd_compression_format.md
 CC ?= gcc
-CFLAGS ?= -O3 
+CFLAGS ?= -O3
 CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99
 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
 LDFLAGS = $(LOC)
@@ -50,7 +50,7 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
 
 .c.o:
 	$(CC) $(CFLAGS) -c -o $@ $<
- 
+
 example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
 	$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
 
@@ -75,10 +75,10 @@ $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwra
 	$(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
 
 $(ZSTDLIBDIR)/libzstd.a:
-	$(MAKE) -C $(ZSTDLIBDIR) all
+	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
 
 $(ZSTDLIBDIR)/libzstd.so:
-	$(MAKE) -C $(ZSTDLIBDIR) all
+	$(MAKE) -C $(ZSTDLIBDIR) libzstd
 
 clean:
 	-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench