lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

Browse Source
This commit is contained in:
Przemyslaw Skibinski
2016-11-03 11:38:04 +01:00
parent 26306fcacf 861cd06ded
commit 1f60eb6b17
11 changed files with 100 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -22,7 +22,7 @@ matrix:
packages:
- gcc-4.8
- g++-4.8
env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean"
env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest && make -C contrib/pzstd all && make -C contrib/pzstd check && make -C contrib/pzstd clean"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"

14
Makefile
View File

@ -20,28 +20,32 @@ else
VOID = /dev/null
endif
.PHONY: default all zlibwrapper zstd clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
default: libzstd zstd
.PHONY: default
default: lib zstd
.PHONY: all
all:
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@ zstd32
$(MAKE) -C $(TESTDIR) $@ all32
libzstd:
.PHONY: lib
lib:
@$(MAKE) -C $(ZSTDDIR)
zstd:
@$(MAKE) -C $(PRGDIR)
cp $(PRGDIR)/zstd .
.PHONY: zlibwrapper
zlibwrapper:
$(MAKE) -C $(ZWRAPDIR) test
.PHONY: test
test:
$(MAKE) -C $(TESTDIR) $@
.PHONY: clean
clean:
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@ -56,6 +60,8 @@ clean:
#------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
HOST_OS = POSIX
.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
install:
@$(MAKE) -C $(ZSTDDIR) $@
@$(MAKE) -C $(PRGDIR) $@

32
lib/Makefile
View File

@ -26,8 +26,9 @@ INCLUDEDIR=$(PREFIX)/include
CPPFLAGS= -I. -I./common
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wpointer-arith
FLAGS = $(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)
ZSTD_FILES := common/*.c compress/*.c decompress/*.c dictBuilder/*.c
@ -39,7 +40,6 @@ ZSTD_FILES+= legacy/*.c
CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
endif
# OS X linker doesn't support -soname, and use different extension
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
ifeq ($(shell uname), Darwin)
@ -54,22 +54,32 @@ else
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
endif
LIBZSTD = libzstd.$(SHARED_EXT_VER)
.PHONY: default all clean install uninstall
default: clean libzstd
default: lib
all: clean libzstd
all: lib
libzstd: $(ZSTD_FILES)
libzstd.a: ARFLAGS = rcs
libzstd.a: $(ZSTD_FILES)
@echo compiling static library
@$(CC) $(FLAGS) -c $^
@$(AR) rcs $@.a *.o
@$(AR) $(ARFLAGS) $@ *.o
$(LIBZSTD): LDFLAGS += -shared -fPIC
$(LIBZSTD): $(ZSTD_FILES)
@echo compiling dynamic library $(LIBVER)
@$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
@echo creating versioned links
@ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR)
@ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT)
@ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT_MAJOR)
@ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT)
libzstd : $(LIBZSTD)
lib: libzstd.a libzstd
clean:
@rm -f core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
@ -89,7 +99,7 @@ libzstd.pc: libzstd.pc.in
-e 's|@VERSION@|$(VERSION)|' \
$< >$@
install: libzstd libzstd.pc
install: libzstd.a libzstd libzstd.pc
@install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
@install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
@cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)

19
lib/decompress/zstd_decompress.c
View File

@ -193,7 +193,24 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
* Decompression section
***************************************************************/
/* See compression format details in : doc/zstd_compression_format.md */
/*! ZSTD_isFrame() :
* Tells if the content of `buffer` starts with a valid Frame Identifier.
* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
* Note 3 : Skippable Frame Identifiers are considered valid. */
unsigned ZSTD_isFrame(const void* buffer, size_t size)
{
if (size < 4) return 0;
{ U32 const magic = MEM_readLE32(buffer);
if (magic == ZSTD_MAGICNUMBER) return 1;
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
}
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (ZSTD_isLegacy(buffer, size)) return 1;
#endif
return 0;
}
/** ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_frameHeaderSize_prefix.

14
lib/dictBuilder/zdict.c
View File

@ -898,12 +898,14 @@ size_t ZDICT_trainFromBuffer_unsafe(
U32 const nb = MIN(25, dictList[0].pos);
U32 const dictContentSize = ZDICT_dictSize(dictList);
U32 u;
DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize);
DISPLAYLEVEL(3, "list %u best segments \n", nb);
for (u=1; u<=nb; u++) {
U32 pos = dictList[u].pos;
U32 length = dictList[u].length;
U32 printedLength = MIN(40, length);
DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos-1, dictContentSize);
DISPLAYLEVEL(3, "list %u best segments \n", nb-1);
for (u=1; u<nb; u++) {
U32 const pos = dictList[u].pos;
U32 const length = dictList[u].length;
U32 const printedLength = MIN(40, length);
if ((pos > samplesBuffSize) || ((pos + length) > samplesBuffSize))
return ERROR(GENERIC); /* should never happen */
DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
u, length, pos, dictList[u].savings);
ZDICT_printHex((const char*)samplesBuffer+pos, printedLength);

7
lib/zstd.h
View File

@ -433,6 +433,13 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
/*--- Advanced decompression functions ---*/
/*! ZSTD_isFrame() :
* Tells if the content of `buffer` starts with a valid Frame Identifier.
* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
* Note 3 : Skippable Frame Identifiers are considered valid. */
ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
/*! ZSTD_estimateDCtxSize() :
* Gives the potential amount of memory allocated to create a ZSTD_DCtx */
ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);

22
programs/Makefile
View File

@ -30,7 +30,8 @@ endif
CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wpointer-arith
CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
@ -41,7 +42,6 @@ ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
ZSTDDECOMP32_O = $(ZSTDDIR)/decompress/zstd_decompress32.o
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
@ -76,30 +76,27 @@ default: zstd
all: zstd
$(ZSTDDECOMP_O): CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
$(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
ifneq (,$(filter Windows%,$(OS)))
windres\generate_res.bat
endif
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES_FILE) -o $@$(EXT)
$(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT)
$(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
$(CC) -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
zstd32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd32 : $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
ifneq (,$(filter Windows%,$(OS)))
windres\generate_res.bat
endif
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ $(RES32_FILE) -o $@$(EXT)
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
zstd_nolegacy : clean_decomp_o
zstd-nolegacy : clean_decomp_o
$(MAKE) zstd ZSTD_LEGACY_SUPPORT=0
zstd-pgo : MOREFLAGS = -fprofile-generate
@ -111,7 +108,7 @@ zstd-pgo : clean zstd
./zstd -b7i2 $(PROFILE_WITH)
./zstd -b5 $(PROFILE_WITH)
$(RM) zstd
$(RM) $(ZSTDDIR)/decompress/zstd_decompress.o
$(RM) $(ZSTDDECOMP_O)
$(MAKE) zstd MOREFLAGS=-fprofile-use
zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c
@ -142,7 +139,6 @@ clean:
clean_decomp_o:
@$(RM) $(ZSTDDECOMP_O)
@$(RM) $(ZSTDDECOMP32_O)
#----------------------------------------------------------------------------------

26
programs/fileio.c
View File

@ -8,16 +8,6 @@
*/
/* *************************************
* Tuning options
***************************************/
#ifndef ZSTD_LEGACY_SUPPORT
/* LEGACY_SUPPORT :
* decompressor can decode older formats (starting from zstd 0.1+) */
# define ZSTD_LEGACY_SUPPORT 1
#endif
/* *************************************
* Compiler Options
***************************************/
@ -29,6 +19,7 @@
# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */
#endif
/*-*************************************
* Includes
***************************************/
@ -44,10 +35,6 @@
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zstd.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_legacy.h" /* ZSTD_isLegacy */
#endif
/*-*************************************
* OS-specific Includes
@ -638,7 +625,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
@return : 0 (no error) */
static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize)
{
size_t const blockSize = MIN (64 KB, bufferSize);
size_t const blockSize = MIN(64 KB, bufferSize);
size_t readFromInput = 1;
unsigned storedSkips = 0;
@ -686,12 +673,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
}
readSomething = 1; /* there is at least >= 4 bytes in srcFile */
if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
{ U32 const magic = MEM_readLE32(ress.srcBuffer);
if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) & (magic != ZSTD_MAGICNUMBER)
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
& (!ZSTD_isLegacy(ress.srcBuffer, toRead))
#endif
) {
if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) { /* pass-through mode */
unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */
@ -700,7 +682,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
fclose(srcFile);
return 1;
} } }
} }
filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead);
}

6
tests/Makefile
View File

@ -76,7 +76,7 @@ zstd:
zstd32:
$(MAKE) -C $(PRGDIR) $@
zstd_nolegacy:
zstd-nolegacy:
$(MAKE) -C $(PRGDIR) $@
fullbench : $(ZSTD_FILES) $(PRGDIR)/datagen.c fullbench.c
@ -181,8 +181,8 @@ test-zstd: zstd zstd-playTests
test-zstd32: ZSTD = $(PRGDIR)/zstd32
test-zstd32: zstd32 zstd-playTests
test-zstd_nolegacy: ZSTD = $(PRGDIR)/zstd
test-zstd_nolegacy: zstd_nolegacy zstd-playTests
test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd
test-zstd-nolegacy: zstd-nolegacy zstd-playTests
test-fullbench: fullbench datagen
./fullbench -i1

6
zlibWrapper/.gitignore vendored
View File

@ -7,6 +7,12 @@ fitblk_zstd.*
zwrapbench.*
foo.gz
example
example_zstd
fitblk
fitblk_zstd
zwrapbench
# Misc files
*.bat
*.zip

4
zlibWrapper/Makefile
View File

@ -75,10 +75,10 @@ $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwra
$(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
$(ZSTDLIBDIR)/libzstd.a:
$(MAKE) -C $(ZSTDLIBDIR) all
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
$(ZSTDLIBDIR)/libzstd.so:
$(MAKE) -C $(ZSTDLIBDIR) all
$(MAKE) -C $(ZSTDLIBDIR) libzstd
clean:
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench