From 360238733a50e3c5397b0665f6653d976f7d0e43 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 21 Sep 2017 11:29:35 -0700 Subject: [PATCH 1/4] Adds LZ4 support by default if LZ4 is available Simple makefile change + quick typename change Test: make clean make # successfully produces binary without lz4 support make clean # with flags to pick up my lz4 build make MOREFLAGS="-L/home/felixh/prog/lz4/lib -I/home/felixh/prog/lz4/lib" # successfully produces binary with lz4 support echo "TEST TEST TEST THIS IS A TEST STRING PLEASE TEST THIS PLEASE OK THANK YOU" | \ ./lz4/lz4 | \ LD_LIBRARY_PATH=/home/felixh/prog/lz4/lib ./zstd/zstd -d # successfully prints TEST TEST TEST THIS IS A TEST STRING PLEASE TEST THIS PLEASE OK THANK YOU --- programs/Makefile | 11 ++++------- programs/fileio.c | 1 + 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index b13629df9..7aa4331ec 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -140,13 +140,10 @@ allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) -zstd zstd4 : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) -zstd zstd4 : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) -zstd4 : CPPFLAGS += $(LZ4CPP) -zstd4 : LDFLAGS += $(LZ4LD) -zstd : LZ4_MSG := - lz4 support is disabled -zstd zstd4 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -zstd zstd4 : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o +zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP) +zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) +zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) +zstd : $(ZSTDLIB_FILES) zstdcli.o fileio.o bench.o datagen.o dibio.o @echo "$(THREAD_MSG)" @echo "$(ZLIB_MSG)" @echo "$(LZMA_MSG)" diff --git a/programs/fileio.c b/programs/fileio.c index 623c4f4df..b43994888 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -56,6 +56,7 @@ #define LZ4_MAGICNUMBER 0x184D2204 #if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS) +# define LZ4F_ENABLE_OBSOLETE_ENUMS # include # include #endif From 5705d9f25a48c6af18d7e2b8430d99b64540c90d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 21 Sep 2017 17:30:43 -0700 Subject: [PATCH 2/4] Add basic tests for the lz4 integration --- .travis.yml | 2 ++ Makefile | 5 +++++ tests/Makefile | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/.travis.yml b/.travis.yml index a52d57af3..26aeab90c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,8 @@ matrix: - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' + - env: Cmd='make lz4install && make -C tests test-lz4' + git: depth: 1 diff --git a/Makefile b/Makefile index e8bdcea33..13e825021 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,7 @@ clean: @$(MAKE) -C examples/ $@ > $(VOID) @$(MAKE) -C contrib/gen_html $@ > $(VOID) @$(RM) zstd$(EXT) zstdmt$(EXT) tmp* + @$(RM) -r lz4 @echo Cleaning completed #------------------------------------------------------------------------------ @@ -274,6 +275,10 @@ gpp6install: apt-add-repo clang38install: APT_PACKAGES="clang-3.8" $(MAKE) apt-install +# Ubuntu 14.04 ships a too-old lz4 +lz4install: + [ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install + endif diff --git a/tests/Makefile b/tests/Makefile index 2746c1392..f30398c06 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -374,4 +374,26 @@ test-decodecorpus-cli: decodecorpus test-pool: poolTests $(QEMU_SYS) ./poolTests +test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd +test-lz4: zstd decodecorpus + ./decodecorpus -ptmp + # lz4 -> zstd + lz4 < tmp | \ + $(ZSTD) -d | \ + cmp - tmp + # zstd -> lz4 + $(ZSTD) --format=lz4 < tmp | \ + lz4 -d | \ + cmp - tmp + # zstd -> zstd + $(ZSTD) --format=lz4 < tmp | \ + $(ZSTD) -d | \ + cmp - tmp + # zstd -> zstd + $(ZSTD) < tmp | \ + $(ZSTD) -d | \ + cmp - tmp + + rm tmp + endif From d0519d4b0cc88ae965bcfffb2efbee7f2eda425d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 28 Sep 2017 19:18:15 -0400 Subject: [PATCH 3/4] Add CLI Program Name Detection for LZ4 --- programs/zstdcli.c | 4 ++++ tests/Makefile | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 89e97c294..20a1b2171 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -61,6 +61,8 @@ #define ZSTD_UNLZMA "unlzma" #define ZSTD_XZ "xz" #define ZSTD_UNXZ "unxz" +#define ZSTD_LZ4 "lz4" +#define ZSTD_UNLZ4 "unlz4" #define KB *(1 <<10) #define MB *(1 <<20) @@ -425,6 +427,8 @@ int main(int argCount, const char* argv[]) if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma */ if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz */ + if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); } /* behave like xz */ + if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); FIO_setRemoveSrcFile(1); } /* behave like unxz */ memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */ diff --git a/tests/Makefile b/tests/Makefile index f30398c06..0741170c7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -375,16 +375,27 @@ test-pool: poolTests $(QEMU_SYS) ./poolTests test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd +test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4 +test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4 test-lz4: zstd decodecorpus + ln -s $(PRGDIR)/zstd lz4 + ln -s $(PRGDIR)/zstd unlz4 + ./decodecorpus -ptmp # lz4 -> zstd lz4 < tmp | \ $(ZSTD) -d | \ cmp - tmp + lz4 < tmp | \ + $(ZSTD_UNLZ4) | \ + cmp - tmp # zstd -> lz4 $(ZSTD) --format=lz4 < tmp | \ lz4 -d | \ cmp - tmp + $(ZSTD_LZ4) < tmp | \ + lz4 -d | \ + cmp - tmp # zstd -> zstd $(ZSTD) --format=lz4 < tmp | \ $(ZSTD) -d | \ @@ -394,6 +405,6 @@ test-lz4: zstd decodecorpus $(ZSTD) -d | \ cmp - tmp - rm tmp + rm tmp lz4 unlz4 endif From dc27c36495a01aae9802cef57ad666bcd4cd08e5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 28 Sep 2017 19:34:39 -0400 Subject: [PATCH 4/4] Update documentation to reflect other format support --- programs/README.md | 10 ++++++++++ programs/zstd.1 | 14 +++++++++++--- programs/zstd.1.md | 4 ++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/programs/README.md b/programs/README.md index 2aae52588..2da9a6db1 100644 --- a/programs/README.md +++ b/programs/README.md @@ -40,6 +40,16 @@ There are however other Makefile targets that create different variations of CLI In which case, linking stage will fail if `lzma` library cannot be found. This might be useful to prevent silent feature disabling. +- __HAVE_LZ4__ : `zstd` can compress and decompress files in `.lz4` formats. + This is ordered through commands `--format=lz4`. + Alternatively, symlinks named `lz4`, or `unlz4` will mimic intended behavior. + `.lz4` support is automatically enabled when `lz4` library is detected at build time. + It's possible to disable `.lz4` support, by setting HAVE_LZ4=0 . + Example : make zstd HAVE_LZ4=0 + It's also possible to force compilation with lz4 support, using HAVE_LZ4=1. + In which case, linking stage will fail if `lz4` library cannot be found. + This might be useful to prevent silent feature disabling. + - __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`. Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible. But older versions (< v0.8.0) produced different, incompatible, frames. diff --git a/programs/zstd.1 b/programs/zstd.1 index 0fad1d277..423195203 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -60,11 +60,11 @@ In most places where an integer argument is expected, an optional suffix is supp . .TP \fBKiB\fR -Multiply the integer by 1,024 (2^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accepted as synonyms for \fBKiB\fR\. +Multiply the integer by 1,024 (2\e \fBKi\fR, \fBK\fR, and \fBKB\fR are accepted as synonyms for \fBKiB\fR\. . .TP \fBMiB\fR -Multiply the integer by 1,048,576 (2^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\. +Multiply the integer by 1,048,576 (2\e \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\. . .SS "Operation mode" If multiple operation mode options are given, the last one takes effect\. @@ -148,6 +148,10 @@ keep source file(s) after successful compression or decompression\. This is the operate recursively on dictionaries . .TP +\fB\-\-format=FORMAT\fR +compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\. +. +.TP \fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR display help/long help and exit . @@ -190,6 +194,10 @@ Dictionary saved into \fBfile\fR (default name: dictionary)\. Limit dictionary to specified size (default: 112640)\. . .TP +\fB\-B#\fR +Split input files in blocks of size # (default: no split) +. +.TP \fB\-\-dictID=#\fR A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. . @@ -340,7 +348,7 @@ Bigger hash tables usually improve compression ratio at the expense of more memo The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\. . .TP -\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmSlen\fR=\fIldmslen\fR +\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR Specify the minimum searched length of a match for long distance matching\. . .IP diff --git a/programs/zstd.1.md b/programs/zstd.1.md index e446422b6..b10c1c4bb 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -143,6 +143,10 @@ the last one takes effect. This is the default behavior. * `-r`: operate recursively on dictionaries +* `--format=FORMAT`: + compress and decompress in other formats. If compiled with + support, zstd can compress to or decompress from other compression algorithm + formats. Possibly available options are `gzip`, `xz`, `lzma`, and `lz4`. * `-h`/`-H`, `--help`: display help/long help and exit * `-V`, `--version`: