From ef4df0df4af33a5fb822509933398e22cfde07d3 Mon Sep 17 00:00:00 2001 From: Ryan Schmidt Date: Fri, 16 Nov 2018 02:16:21 -0600 Subject: [PATCH 01/62] Fix i386 build failure "Junk character 13" --- lib/common/cpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common/cpu.h b/lib/common/cpu.h index eeb428ad5..5f0923fc9 100644 --- a/lib/common/cpu.h +++ b/lib/common/cpu.h @@ -78,7 +78,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { __asm__( "pushl %%ebx\n\t" "cpuid\n\t" - "movl %%ebx, %%eax\n\r" + "movl %%ebx, %%eax\n\t" "popl %%ebx" : "=a"(f7b), "=c"(f7c) : "a"(7), "c"(0) From 46d470c20cc981fb87f4c3b2742f365c292de5d7 Mon Sep 17 00:00:00 2001 From: Ryan Schmidt Date: Fri, 16 Nov 2018 03:49:15 -0600 Subject: [PATCH 02/62] Fix feature detection with multiple -arch flags When multiple -arch flags are used, the compiler invokes itself once for each architecture. Apparently, input on stdin is consumed by the compilation of the first arch and is no longer available to the compilation of the second arch, which results in a build failure and the potentially incorrect determination that a feature is not available. So write the feature detection source to a file instead of using stdin. --- programs/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 77c1d6a2d..6249e5921 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -91,7 +91,7 @@ VOID = /dev/null # thread detection NO_THREAD_MSG := ==> no threads, building without multithreading support -HAVE_PTHREAD := $(shell printf '\#include \nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_pthread$(EXT) -x c - -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0) +HAVE_PTHREAD := $(shell printf '\#include \nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c) HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0) ifeq ($(HAVE_THREAD), 1) THREAD_MSG := ==> building with threading support @@ -103,7 +103,7 @@ endif # zlib detection NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support -HAVE_ZLIB := $(shell printf '\#include \nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_zlib$(EXT) -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0) +HAVE_ZLIB := $(shell printf '\#include \nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c) ifeq ($(HAVE_ZLIB), 1) ZLIB_MSG := ==> building zstd with .gz compression support ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS @@ -114,7 +114,7 @@ endif # lzma detection NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support -HAVE_LZMA := $(shell printf '\#include \nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lzma$(EXT) -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0) +HAVE_LZMA := $(shell printf '\#include \nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c) ifeq ($(HAVE_LZMA), 1) LZMA_MSG := ==> building zstd with .xz/.lzma compression support LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS @@ -125,7 +125,7 @@ endif # lz4 detection NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support -HAVE_LZ4 := $(shell printf '\#include \n\#include \nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lz4$(EXT) -x c - -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0) +HAVE_LZ4 := $(shell printf '\#include \n\#include \nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c) ifeq ($(HAVE_LZ4), 1) LZ4_MSG := ==> building zstd with .lz4 compression support LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS From 2abd5139a562a5dd84dc0a06b8c2a7ab6e3ce24f Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Tue, 27 Nov 2018 00:24:33 +0700 Subject: [PATCH 03/62] Add meson build guide --- contrib/meson/README | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/contrib/meson/README b/contrib/meson/README index 0b5331e6d..6641f64c9 100644 --- a/contrib/meson/README +++ b/contrib/meson/README @@ -1,3 +1,12 @@ This Meson project is provided with no guarantee and maintained by Dima Krasner . It outputs one libzstd, either shared or static, depending on default_library. + +How to build +============ + +`cd` to this meson directory (`zstd/contrib/meson`) and type: + + $ meson --buildtype=release --strip --prefix=/usr builddir + $ ninja # to build + $ ninja install # to install From 9bd8f6a00cd3ab07e61e63db99e3b0c04380811b Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 00:10:04 +0700 Subject: [PATCH 04/62] Rename and update build instruction in README file to README.md --- contrib/meson/README | 12 ------------ contrib/meson/README.md | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 12 deletions(-) delete mode 100644 contrib/meson/README create mode 100644 contrib/meson/README.md diff --git a/contrib/meson/README b/contrib/meson/README deleted file mode 100644 index 6641f64c9..000000000 --- a/contrib/meson/README +++ /dev/null @@ -1,12 +0,0 @@ -This Meson project is provided with no guarantee and maintained by Dima Krasner . - -It outputs one libzstd, either shared or static, depending on default_library. - -How to build -============ - -`cd` to this meson directory (`zstd/contrib/meson`) and type: - - $ meson --buildtype=release --strip --prefix=/usr builddir - $ ninja # to build - $ ninja install # to install diff --git a/contrib/meson/README.md b/contrib/meson/README.md new file mode 100644 index 000000000..a2730faaa --- /dev/null +++ b/contrib/meson/README.md @@ -0,0 +1,31 @@ +This Meson project is provided with no guarantee and maintained +by Dima Krasner . + +It outputs one `libzstd`, either shared or static, depending on +`default_library` option. + +How to build +============ + +`cd` to this meson directory (`zstd/contrib/meson`) and type: + +```sh +meson --buildtype=release --strip --prefix=/usr builddir +cd builddir +ninja # to build +ninja install # to install +``` + +You might want to install it in staging directory: + +```sh +DESTDIR=./staging ninja install +``` + +To configure the build, use: + +```sh +meson configure +``` + +See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html). From 9a721e5216f0276d130067e4881e87a9bc4432a2 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 01:04:41 +0700 Subject: [PATCH 05/62] Update meson build system NOTE: This commit only tested on Linux (Ubuntu 18.04). Windows build may not work as expected. * Use meson >= 0.47.0 cause we use install_man function * Add three helper Python script: * CopyFile.py: To copy file * CreateSymlink.py: To make symlink (both Windows and Unix) * GetZstdLibraryVersion.py: Parse lib/zstd.h to get zstd version These help emulating equivalent functions in CMake and Makefile. * Use subdir from meson to split meson.build * Add contrib build * Fix other build * Add new build options * build_programs: Enable programs build * build_contrib: Enable contrib build * build_tests: Enable tests build * use_static_runtime: Link to static run-time libraries on MSVC * zlib_support: Enable zlib support * lzma_support: Enable lzma support --- contrib/meson/CopyFile.py | 35 ++++ contrib/meson/CreateSymlink.py | 36 ++++ contrib/meson/GetZstdLibraryVersion.py | 45 ++++ contrib/meson/contrib/gen_html/meson.build | 36 ++++ contrib/meson/contrib/meson.build | 12 ++ contrib/meson/contrib/pzstd/meson.build | 32 +++ contrib/meson/lib/meson.build | 122 +++++++++++ contrib/meson/meson.build | 227 +++++++++------------ contrib/meson/meson_options.txt | 27 ++- contrib/meson/programs/meson.build | 113 ++++++++++ contrib/meson/tests/meson.build | 65 ++++++ 11 files changed, 617 insertions(+), 133 deletions(-) create mode 100644 contrib/meson/CopyFile.py create mode 100644 contrib/meson/CreateSymlink.py create mode 100644 contrib/meson/GetZstdLibraryVersion.py create mode 100644 contrib/meson/contrib/gen_html/meson.build create mode 100644 contrib/meson/contrib/meson.build create mode 100644 contrib/meson/contrib/pzstd/meson.build create mode 100644 contrib/meson/lib/meson.build create mode 100644 contrib/meson/programs/meson.build create mode 100644 contrib/meson/tests/meson.build diff --git a/contrib/meson/CopyFile.py b/contrib/meson/CopyFile.py new file mode 100644 index 000000000..6c0288ee4 --- /dev/null +++ b/contrib/meson/CopyFile.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# ############################################################################# +# Copyright (c) 2018-present lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# +import os +import sys +import shutil + + +def usage(): + print('usage: python3 CreateSymlink.py ') + print('Copy the file named src to a file named dst') + sys.exit(1) + + +def main(): + if len(sys.argv) < 3: + usage() + src = sys.argv[1] + dst = sys.argv[2] + + if os.path.exists(dst): + print ('File already exists: %r' % (dst)) + return + + shutil.copy2(src, dst) + + +if __name__ == '__main__': + main() diff --git a/contrib/meson/CreateSymlink.py b/contrib/meson/CreateSymlink.py new file mode 100644 index 000000000..d0f9918ab --- /dev/null +++ b/contrib/meson/CreateSymlink.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# ############################################################################# +# Copyright (c) 2018-present lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# +import os +import sys + + +def usage(): + print('usage: python3 CreateSymlink.py [dst is dir: True or False]') + sys.exit(1) + + +def main(): + if len(sys.argv) < 3: + usage() + src = sys.argv[1] + dst = sys.argv[2] + is_dir = False + if len(sys.argv) == 4: + is_dir = bool(sys.argv[3]) + + if os.path.islink(dst) and os.readlink(dst) == src: + print ('File exists: %r -> %r' % (dst, src)) + return + + os.symlink(src, dst, is_dir) + + +if __name__ == '__main__': + main() diff --git a/contrib/meson/GetZstdLibraryVersion.py b/contrib/meson/GetZstdLibraryVersion.py new file mode 100644 index 000000000..364edc5aa --- /dev/null +++ b/contrib/meson/GetZstdLibraryVersion.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# ############################################################################# +# Copyright (c) 2018-present lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# +import re +import sys + + +def usage(): + print('usage: python3 GetZstdLibraryVersion.py ') + sys.exit(1) + + +def find_version(filepath): + version_file_data = None + with open(filepath) as fd: + version_file_data = fd.read() + + patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+) +#\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+) +#\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+) +""" + regex = re.compile(patterns, re.MULTILINE) + version_match = regex.search(version_file_data) + if version_match: + return version_match.groups() + raise RuntimeError("Unable to find version string.") + + +def main(): + if len(sys.argv) < 2: + usage() + + filepath = sys.argv[1] + version_tup = find_version(filepath) + print('.'.join(version_tup)) + + +if __name__ == '__main__': + main() diff --git a/contrib/meson/contrib/gen_html/meson.build b/contrib/meson/contrib/gen_html/meson.build new file mode 100644 index 000000000..086b8f058 --- /dev/null +++ b/contrib/meson/contrib/gen_html/meson.build @@ -0,0 +1,36 @@ +# ############################################################################# +# Copyright (c) 2018-present lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# + +zstd_source_dir = join_paths('..', '..', '..', '..') +library_dir = join_paths(zstd_source_dir, 'lib') +library_common_dir = join_paths(library_dir, 'common') +programs_dir = join_paths(zstd_source_dir, 'programs') +contrib_gen_html_dir = join_paths(zstd_source_dir, 'contrib', 'gen_html') + +gen_html_includes = include_directories(programs_dir, + library_dir, + library_common_dir, + contrib_gen_html_dir ) + +gen_html = executable('gen_html', + join_paths(contrib_gen_html_dir, 'gen_html.cpp'), + include_directories: gen_html_includes, + install: false ) + +# Update zstd manual +zstd_manual_html = custom_target('zstd_manual.html', + output : 'zstd_manual.html', + command : [ + gen_html, + zstd_version, + join_paths(meson.current_source_dir(), library_dir, 'zstd.h'), + '@OUTPUT@' + ], + install : true, + install_dir : zstd_docdir ) diff --git a/contrib/meson/contrib/meson.build b/contrib/meson/contrib/meson.build new file mode 100644 index 000000000..7f6d03a4c --- /dev/null +++ b/contrib/meson/contrib/meson.build @@ -0,0 +1,12 @@ +# ############################################################################# +# Copyright (c) 2018-present Dima Krasner +# lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# + +subdir('pzstd') +subdir('gen_html') diff --git a/contrib/meson/contrib/pzstd/meson.build b/contrib/meson/contrib/pzstd/meson.build new file mode 100644 index 000000000..923990145 --- /dev/null +++ b/contrib/meson/contrib/pzstd/meson.build @@ -0,0 +1,32 @@ +# ############################################################################# +# Copyright (c) 2018-present lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# + +zstd_source_dir = join_paths('..', '..', '..', '..') +library_dir = join_paths(zstd_source_dir, 'lib') +library_common_dir = join_paths(library_dir, 'common') +programs_dir = join_paths(zstd_source_dir, 'programs') +contrib_pzstd_dir = join_paths(zstd_source_dir, 'contrib', 'pzstd') + +pzstd_includes = include_directories(programs_dir, + library_dir, + library_common_dir, + contrib_pzstd_dir) +pzstd_sources = [join_paths(programs_dir, 'util.c'), + join_paths(contrib_pzstd_dir, 'main.cpp'), + join_paths(contrib_pzstd_dir, 'Options.cpp'), + join_paths(contrib_pzstd_dir, 'Pzstd.cpp'), + join_paths(contrib_pzstd_dir, 'SkippableFrame.cpp')] + +pzstd = executable('pzstd', + pzstd_sources, + cpp_args: [ '-DNDEBUG', '-Wno-shadow' ], + include_directories: pzstd_includes, + link_with: libzstd, + dependencies: [ thread_dep ], + install: true) diff --git a/contrib/meson/lib/meson.build b/contrib/meson/lib/meson.build new file mode 100644 index 000000000..b5d6778dd --- /dev/null +++ b/contrib/meson/lib/meson.build @@ -0,0 +1,122 @@ +# ############################################################################# +# Copyright (c) 2018-present Dima Krasner +# lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# + +zstd_source_dir = join_paths('..', '..', '..') +library_dir = join_paths(zstd_source_dir, 'lib') +library_common_dir = join_paths(library_dir, 'common') +library_compress_dir = join_paths(library_dir, 'compress') +library_decompress_dir = join_paths(library_dir, 'decompress') +library_dictbuilder_dir = join_paths(library_dir, 'dictBuilder') +library_deprecated_dir = join_paths(library_dir, 'deprecated') +library_legacy_dir = join_paths(library_dir, 'legacy') + +libzstd_includes = [include_directories(library_dir, + library_common_dir, + library_compress_dir, + library_decompress_dir, + library_dictbuilder_dir, + library_deprecated_dir)] + +libzstd_sources = [join_paths(library_common_dir, 'entropy_common.c'), + join_paths(library_common_dir, 'fse_decompress.c'), + join_paths(library_common_dir, 'threading.c'), + join_paths(library_common_dir, 'pool.c'), + join_paths(library_common_dir, 'zstd_common.c'), + join_paths(library_common_dir, 'error_private.c'), + join_paths(library_common_dir, 'xxhash.c'), + join_paths(library_compress_dir, 'hist.c'), + join_paths(library_compress_dir, 'fse_compress.c'), + join_paths(library_compress_dir, 'huf_compress.c'), + join_paths(library_compress_dir, 'zstd_compress.c'), + join_paths(library_compress_dir, 'zstdmt_compress.c'), + join_paths(library_compress_dir, 'zstd_fast.c'), + join_paths(library_compress_dir, 'zstd_double_fast.c'), + join_paths(library_compress_dir, 'zstd_lazy.c'), + join_paths(library_compress_dir, 'zstd_opt.c'), + join_paths(library_compress_dir, 'zstd_ldm.c'), + join_paths(library_decompress_dir, 'huf_decompress.c'), + join_paths(library_decompress_dir, 'zstd_decompress.c'), + join_paths(library_decompress_dir, 'zstd_decompress_block.c'), + join_paths(library_decompress_dir, 'zstd_ddict.c'), + join_paths(library_dictbuilder_dir, 'cover.c'), + join_paths(library_dictbuilder_dir, 'fastcover.c'), + join_paths(library_dictbuilder_dir, 'divsufsort.c'), + join_paths(library_dictbuilder_dir, 'zdict.c'), + join_paths(library_deprecated_dir, 'zbuff_common.c'), + join_paths(library_deprecated_dir, 'zbuff_compress.c'), + join_paths(library_deprecated_dir, 'zbuff_decompress.c')] + +if legacy_support == '0' + legacy_support = 'false' +endif +if legacy_support != 'false' + if legacy_support == 'true' + legacy_support = '1' + endif + legacy_int = legacy_support.to_int() + if legacy_int < 0 or legacy_int >= 8 + legacy_int = 0 + endif + add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_int), + language: 'c') + libzstd_includes += [ include_directories(library_legacy_dir) ] + # See ZSTD_LEGACY_SUPPORT of programs/README.md + message('Enable legacy support back to version 0.@0@'.format(legacy_int)) + if legacy_int <= 1 + libzstd_sources += join_paths(library_legacy_dir, 'zstd_v01.c') + endif + if legacy_int <= 2 + libzstd_sources += join_paths(library_legacy_dir, 'zstd_v02.c') + endif + if legacy_int <= 3 + libzstd_sources += join_paths(library_legacy_dir, 'zstd_v03.c') + endif + if legacy_int <= 4 + libzstd_sources += join_paths(library_legacy_dir, 'zstd_v04.c') + endif + if legacy_int <= 5 + libzstd_sources += join_paths(library_legacy_dir, 'zstd_v05.c') + endif + if legacy_int <= 6 + libzstd_sources += join_paths(library_legacy_dir, 'zstd_v06.c') + endif + if legacy_int <= 7 + libzstd_sources += join_paths(library_legacy_dir, 'zstd_v07.c') + endif +endif + +if enable_multithread + message('Enable multi-threading support') + add_project_arguments('-DZSTD_MULTITHREAD', language: 'c') + libzstd_deps = [ thread_dep ] +else + libzstd_deps = [] +endif + +libzstd = library('zstd', + libzstd_sources, + include_directories: libzstd_includes, + dependencies: libzstd_deps, + install: true, + soversion: '1') + +pkgconfig.generate(name: 'libzstd', + description: 'fast lossless compression algorithm library', + version: zstd_version, + filebase: 'libzstd', + libraries: [libzstd], + #subdirs: ['.'] + ) + +install_headers(join_paths(library_dir, 'zstd.h'), + join_paths(library_deprecated_dir, 'zbuff.h'), + join_paths(library_dictbuilder_dir, 'zdict.h'), + join_paths(library_dictbuilder_dir, 'cover.h'), + join_paths(library_common_dir, 'zstd_errors.h')) diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index 98c9b0293..b9ac88281 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -1,144 +1,109 @@ -project('zstd', 'c', license: 'BSD') +# ############################################################################# +# Copyright (c) 2018-present Dima Krasner +# lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# -libm = meson.get_compiler('c').find_library('m', required: true) +project('zstd', + ['c', 'cpp'], + license: 'BSD', + default_options : ['c_std=c99', + 'cpp_std=c++11', + 'buildtype=release'], + version: '1.3.7', + # for install_man + meson_version: '>=0.47.0') -lib_dir = join_paths('..', '..', 'lib') -common_dir = join_paths(lib_dir, 'common') -compress_dir = join_paths(lib_dir, 'compress') -decompress_dir = join_paths(lib_dir, 'decompress') -dictbuilder_dir = join_paths(lib_dir, 'dictBuilder') -deprecated_dir = join_paths(lib_dir, 'deprecated') +cc = meson.get_compiler('c') +cxx = meson.get_compiler('cpp') +pkgconfig = import('pkgconfig') +python3 = import('python').find_installation() -libzstd_srcs = [ - join_paths(common_dir, 'entropy_common.c'), - join_paths(common_dir, 'fse_decompress.c'), - join_paths(common_dir, 'threading.c'), - join_paths(common_dir, 'pool.c'), - join_paths(common_dir, 'zstd_common.c'), - join_paths(common_dir, 'error_private.c'), - join_paths(common_dir, 'xxhash.c'), - join_paths(compress_dir, 'fse_compress.c'), - join_paths(compress_dir, 'hist.c'), - join_paths(compress_dir, 'huf_compress.c'), - join_paths(compress_dir, 'zstd_compress.c'), - join_paths(compress_dir, 'zstd_fast.c'), - join_paths(compress_dir, 'zstd_double_fast.c'), - join_paths(compress_dir, 'zstd_lazy.c'), - join_paths(compress_dir, 'zstd_opt.c'), - join_paths(compress_dir, 'zstd_ldm.c'), - join_paths(compress_dir, 'zstdmt_compress.c'), - join_paths(decompress_dir, 'huf_decompress.c'), - join_paths(decompress_dir, 'zstd_decompress.c'), - join_paths(dictbuilder_dir, 'cover.c'), - join_paths(dictbuilder_dir, 'divsufsort.c'), - join_paths(dictbuilder_dir, 'zdict.c'), - join_paths(deprecated_dir, 'zbuff_common.c'), - join_paths(deprecated_dir, 'zbuff_compress.c'), - join_paths(deprecated_dir, 'zbuff_decompress.c') -] +zstd_version = meson.project_version() -libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)] +# ============================================================================= +# Project directories +# ============================================================================= -legacy = get_option('legacy_support') -if legacy == '0' - legacy = 'false' -endif -if legacy != 'false' - if legacy == 'true' - legacy = '1' - endif - #See ZSTD_LEGACY_SUPPORT of programs/README.md - message('Enabling legacy support back to version 0.' + legacy) - legacy_int = legacy.to_int() - if legacy_int > 7 - legacy_int = 7 - endif - libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=' + legacy] +zstd_prefix = get_option('prefix') +zstd_bindir = join_paths(zstd_prefix, get_option('bindir')) +zstd_datadir = join_paths(zstd_prefix, get_option('datadir')) +zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name()) +zstd_mandir = join_paths(zstd_prefix, get_option('mandir')) - legacy_dir = join_paths(lib_dir, 'legacy') - libzstd_includes += [include_directories(legacy_dir)] - if legacy_int <= 1 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v01.c') +zstd_source_dir = join_paths('..', '..') +library_dir = join_paths(zstd_source_dir, 'lib') +contrib_meson_dir = join_paths(zstd_source_dir, 'contrib', 'meson') + +# ============================================================================= +# Project options +# ============================================================================= + +legacy_support = get_option('legacy_support') +enable_programs = get_option('build_programs') +enable_tests = get_option('build_tests') +enable_contrib = get_option('build_contrib') +enable_multithread = get_option('multithread_support') +enable_zlib = get_option('zlib_support') +enable_lzma = get_option('lzma_support') + +# ============================================================================= +# Getting project version from zstd.h +# ============================================================================= + +GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py') +zstd_h_file = join_paths(library_dir, 'zstd.h') +r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file) +if r.returncode() == 0 + output = r.stdout().strip() + if output.version_compare('>@0@'.format(zstd_version)) + zstd_version = output + message('Project version is now: @0@'.format(zstd_version)) endif - if legacy_int <= 2 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v02.c') - endif - if legacy_int <= 3 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v03.c') - endif - if legacy_int <= 4 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v04.c') - endif - if legacy_int <= 5 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v05.c') - endif - if legacy_int <= 6 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v06.c') - endif - if legacy_int <= 7 - libzstd_srcs += join_paths(legacy_dir, 'zstd_v07.c') - endif -else - libzstd_cflags = [] endif -if get_option('multithread') - message('Enabling multi-threading support') - add_global_arguments('-DZSTD_MULTITHREAD', language: 'c') - libzstd_deps = [dependency('threads')] -else - libzstd_deps = [] +# ============================================================================= +# Dependencies +# ============================================================================= + +libm_dep = cc.find_library('m', required: true) +thread_dep = dependency('threads', required: false) +zlib_dep = dependency('zlib', required: false) +lzma_dep = dependency('lzma', required: false) + +# ============================================================================= +# Compiler flags +# ============================================================================= + +if cxx.get_id() == 'gcc' or cxx.get_id() == 'clang' + common_flags = [ '-DXXH_NAMESPACE=ZSTD_' ] + zstd_compilation_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ] + cc_common_flags = cc.get_supported_arguments(zstd_compilation_flags) + cc_common_flags += cc.get_supported_arguments(['-Wstrict-prototypes']) + cc_common_flags += common_flags + cxx_common_flags = cxx.get_supported_arguments(zstd_compilation_flags) + cxx_common_flags += common_flags + add_project_arguments(cc_common_flags, language : 'c') + add_project_arguments(cxx_common_flags, language : 'cpp') endif -libzstd = library('zstd', - libzstd_srcs, - include_directories: libzstd_includes, - c_args: libzstd_cflags, - dependencies: libzstd_deps, - install: true, - soversion: '1', - ) +# ============================================================================= +# Subdirs +# ============================================================================= -programs_dir = join_paths('..', '..', 'programs') - -zstd = executable('zstd', - join_paths(programs_dir, 'bench.c'), - join_paths(programs_dir, 'datagen.c'), - join_paths(programs_dir, 'dibio.c'), - join_paths(programs_dir, 'fileio.c'), - join_paths(programs_dir, 'zstdcli.c'), - include_directories: libzstd_includes, - c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'], - link_with: libzstd, - install: true) - -tests_dir = join_paths('..', '..', 'tests') -datagen_c = join_paths(programs_dir, 'datagen.c') -test_includes = libzstd_includes + [include_directories(programs_dir)] - -fullbench = executable('fullbench', - datagen_c, join_paths(tests_dir, 'fullbench.c'), - include_directories: test_includes, - link_with: libzstd) -test('fullbench', fullbench) - -fuzzer = executable('fuzzer', - datagen_c, join_paths(tests_dir, 'fuzzer.c'), - include_directories: test_includes, - link_with: libzstd) -test('fuzzer', fuzzer) - -if target_machine.system() != 'windows' - paramgrill = executable('paramgrill', - datagen_c, join_paths(tests_dir, 'paramgrill.c'), - join_paths(programs_dir, 'bench.c'), - include_directories: test_includes, - link_with: libzstd, - dependencies: libm) - test('paramgrill', paramgrill) - - datagen = executable('datagen', - datagen_c, join_paths(tests_dir, 'datagencli.c'), - include_directories: test_includes, - link_with: libzstd) +subdir('lib') +if enable_programs + subdir('programs') +endif + +if enable_tests + subdir('tests') +endif + +if enable_contrib + subdir('contrib') endif diff --git a/contrib/meson/meson_options.txt b/contrib/meson/meson_options.txt index 99845c8aa..1d28c71a8 100644 --- a/contrib/meson/meson_options.txt +++ b/contrib/meson/meson_options.txt @@ -1,3 +1,26 @@ -option('multithread', type: 'boolean', value: false) +# ############################################################################# +# Copyright (c) 2018-present Dima Krasner +# lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# + +option('multithread_support', type: 'boolean', value: true, + description: 'Enable multithreading when pthread is detected') option('legacy_support', type: 'string', value: '4', - description: 'True or false, or 7 to 1 for v0.7+ to v0.1+.') + description: 'Support any legacy format: true or false, or 7 to 1 for v0.7+ to v0.1+') +option('build_programs', type: 'boolean', value: true, + description: 'Enable programs build') +option('build_contrib', type: 'boolean', value: false, + description: 'Enable contrib build') +option('build_tests', type: 'boolean', value: false, + description: 'Enable tests build') +option('use_static_runtime', type: 'boolean', value: false, + description: 'Link to static run-time libraries on MSVC') +option('zlib_support', type: 'boolean', value: false, + description: 'Enable zlib support') +option('lzma_support', type: 'boolean', value: false, + description: 'Enable lzma support') diff --git a/contrib/meson/programs/meson.build b/contrib/meson/programs/meson.build new file mode 100644 index 000000000..63ea328ae --- /dev/null +++ b/contrib/meson/programs/meson.build @@ -0,0 +1,113 @@ +# ############################################################################# +# Copyright (c) 2018-present Dima Krasner +# lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# + +zstd_source_dir = join_paths('..', '..', '..') +programs_dir = join_paths(zstd_source_dir, 'programs') + +zstdcli_c_file = join_paths(programs_dir, 'zstdcli.c') +util_c_file = join_paths(programs_dir, 'util.c') +fileio_c_file = join_paths(programs_dir, 'fileio.c') +zstd_programs_sources = [zstdcli_c_file, + util_c_file, + fileio_c_file, + join_paths(programs_dir, 'benchfn.c'), + join_paths(programs_dir, 'benchzstd.c'), + join_paths(programs_dir, 'datagen.c'), + join_paths(programs_dir, 'dibio.c')] + +zstd_c_args = [] +if enable_multithread + zstd_c_args += [ '-DZSTD_MULTITHREAD' ] +endif + +zstd_deps = [] +if enable_zlib and zlib_dep.found() + zstd_deps += [ zlib_dep ] + zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ] +endif + +if enable_lzma and lzma_dep.found() + zstd_deps += [ lzma_dep ] + zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ] +endif + +zstd = executable('zstd', + zstd_programs_sources, + c_args: zstd_c_args, + include_directories: libzstd_includes, + link_with: libzstd, + dependencies: zstd_deps, + install: true) + +zstd_frugal_sources = [join_paths(programs_dir, 'zstdcli.c'), + util_c_file, + fileio_c_file] + +executable('zstd-frugal', + zstd_frugal_sources, + include_directories: libzstd_includes, + link_with: libzstd, + c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ], + install: true) + +# ============================================================================= +# Program symlinks +# ============================================================================= + +CreateSymlink_py = join_paths(meson.current_source_dir(), '..', 'CreateSymlink.py') + +foreach f : [ 'zstdcat', 'unzstd' ] + custom_target(f, + output : f, + input: zstd, + command : [python3, CreateSymlink_py, '@PLAINNAME@', '@OUTPUT@'], + build_always_stale: false, + install : true, + install_dir: zstd_bindir) +endforeach + +if enable_multithread + custom_target('zstdmt', + output : 'zstdmt', + input: zstd, + command : [python3, CreateSymlink_py, '@PLAINNAME@', '@OUTPUT@'], + build_always_stale: false, + install : true, + install_dir: zstd_bindir) +endif + +# ============================================================================= +# Manpages +# ============================================================================= + +zstd_man1_dir = join_paths(zstd_mandir, 'man1') +zstd_1_file = join_paths(programs_dir, 'zstd.1') +CopyFile_py = join_paths(meson.current_source_dir(), '..', 'CopyFile.py') + +custom_target('zstd.1', + output : 'zstd.1', + input: zstd_1_file, + command : [python3, CopyFile_py, '@INPUT@', '@OUTPUT@'], + build_always_stale: false, + install : true, + install_dir: zstd_man1_dir) + +foreach f : [ 'zstdcat.1', 'unzstd.1' ] + custom_target(f, + output : f, + input: zstd_1_file, + command : [python3, CreateSymlink_py, '@PLAINNAME@', '@OUTPUT@'], + install : true, + build_always_stale: false, + install_dir: zstd_man1_dir) +endforeach + +install_man(join_paths(programs_dir, 'zstdgrep.1'), + join_paths(programs_dir, 'zstdless.1')) diff --git a/contrib/meson/tests/meson.build b/contrib/meson/tests/meson.build new file mode 100644 index 000000000..cc9e1ee10 --- /dev/null +++ b/contrib/meson/tests/meson.build @@ -0,0 +1,65 @@ +# ############################################################################# +# Copyright (c) 2018-present Dima Krasner +# lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# + +zstd_source_dir = join_paths('..', '..', '..') +programs_dir = join_paths(zstd_source_dir, 'programs') +tests_dir = join_paths(zstd_source_dir, 'tests') + +datagen_c_file = join_paths(programs_dir, 'datagen.c') +util_c_file = join_paths(programs_dir, 'util.c') +benchfn_c_file = join_paths(programs_dir, 'benchfn.c') + +datagen_sources = [datagen_c_file, + join_paths(tests_dir, 'datagencli.c')] +test_includes = libzstd_includes + [include_directories(programs_dir)] + +datagen = executable('datagen', + datagen_sources, + include_directories: test_includes, + link_with: libzstd, + install: false) +test('datagen', datagen) + +fullbench_sources = [datagen_c_file, + util_c_file, + benchfn_c_file, + join_paths(programs_dir, 'benchzstd.c'), + join_paths(programs_dir, 'fullbench.c')] +fullbench = executable('fullbench', + fullbench_sources, + include_directories: test_includes, + link_with: libzstd, + install: false) +test('fullbench', fullbench) + +fuzzer_sources = [datagen_c_file, + util_c_file, + join_paths(tests_dir, 'fuzzer.c')] +fuzzer = executable('fuzzer', + fuzzer_sources, + include_directories: test_includes, + link_with: libzstd, + install: false) +test('fuzzer', fuzzer) + +paramgrill_sources = [benchfn_c_file + join_paths(programs_dir, 'benchzstd.c'), + datagen_c_file + util_c_file + join_paths(tests_dir, 'paramgrill.c')] +if host_machine.system() != 'windows' + paramgrill = executable('paramgrill', + paramgrill_sources, + include_directories: test_includes, + link_with: libzstd, + dependencies: libm_dep, + install: false ) + test('paramgrill', paramgrill) +endif From 71b8ee1bf19dbe5571582c1fb778ed67e028d07e Mon Sep 17 00:00:00 2001 From: Denis Ahrens Date: Wed, 28 Nov 2018 04:51:16 +0100 Subject: [PATCH 06/62] fixed a typo --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9f908355f..9f9bc8420 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -151,7 +151,7 @@ static int usage_advanced(const char* programName) #ifdef UTIL_HAS_CREATEFILELIST DISPLAY( " -r : operate recursively on directories \n"); #endif - DISPLAY( "--format=zstd : compress files to the .zstd format (default) \n"); + DISPLAY( "--format=zstd : compress files to the .zst format (default) \n"); #ifdef ZSTD_GZCOMPRESS DISPLAY( "--format=gzip : compress files to the .gz format \n"); #endif From c046e0b626491c613bb82fb81611d727bc24e0c9 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 02:45:01 +0700 Subject: [PATCH 07/62] Fix #1428 - zstdgrep now returns 1 on unmatch --- programs/zstdgrep | 137 +++++++++++++++++++++++----------------------- 1 file changed, 70 insertions(+), 67 deletions(-) diff --git a/programs/zstdgrep b/programs/zstdgrep index 9f871c03f..7f050d04a 100755 --- a/programs/zstdgrep +++ b/programs/zstdgrep @@ -31,94 +31,97 @@ grep_args="" hyphen=0 silent=0 -prg=$(basename $0) +prg=$(basename "$0") # handle being called 'zegrep' or 'zfgrep' -case ${prg} in - *zegrep) - grep_args="-E";; - *zfgrep) - grep_args="-F";; +case "${prg}" in + *zegrep) grep_args="-E";; + *zfgrep) grep_args="-F";; esac # skip all options and pass them on to grep taking care of options # with arguments, and if -e was supplied -while [ $# -gt 0 -a ${endofopts} -eq 0 ] -do - case $1 in +while [ "$#" -gt 0 ] && [ "${endofopts}" -eq 0 ]; do + case "$1" in # from GNU grep-2.5.1 -- keep in sync! - -[ABCDXdefm]) - if [ $# -lt 2 ] - then - echo "${prg}: missing argument for $1 flag" >&2 - exit 1 - fi - case $1 in - -e) - pattern="$2" - pattern_found=1 - shift 2 - break - ;; - *) - ;; - esac - grep_args="${grep_args} $1 $2" - shift 2 - ;; - --) - shift - endofopts=1 - ;; - -) - hyphen=1 - shift - ;; - -h) - silent=1 - shift - ;; - -*) - grep_args="${grep_args} $1" - shift - ;; - *) - # pattern to grep for - endofopts=1 - ;; + -[ABCDXdefm]) + if [ "$#" -lt 2 ]; then + printf '%s: missing argument for %s flag\n' "${prg}" "$1" >&2 + exit 1 + fi + case "$1" in + -e) + pattern="$2" + pattern_found=1 + shift 2 + break + ;; + *) + ;; + esac + grep_args="${grep_args} $1 $2" + shift 2 + ;; + --) + shift + endofopts=1 + ;; + -) + hyphen=1 + shift + ;; + -h) + silent=1 + shift + ;; + -*) + grep_args="${grep_args} $1" + shift + ;; + *) + # pattern to grep for + endofopts=1 + ;; esac done # if no -e option was found, take next argument as grep-pattern -if [ ${pattern_found} -lt 1 ] -then - if [ $# -ge 1 ]; then - pattern="$1" - shift - elif [ ${hyphen} -gt 0 ]; then - pattern="-" +if [ "${pattern_found}" -lt 1 ]; then + if [ "$#" -ge 1 ]; then + pattern="$1" + shift + elif [ "${hyphen}" -gt 0 ]; then + pattern="-" else - echo "${prg}: missing pattern" >&2 - exit 1 + printf '%s: missing pattern\n' "${prg}" >&2 + exit 1 fi fi +EXIT_CODE=0 # call grep ... -if [ $# -lt 1 ] -then +if [ "$#" -lt 1 ]; then # ... on stdin - ${zcat} -fq - | ${grep} ${grep_args} -- "${pattern}" - + # shellcheck disable=SC2086 + "${zcat}" -fq - | "${grep}" ${grep_args} -- "${pattern}" - + EXIT_CODE=$? else # ... on all files given on the command line - if [ ${silent} -lt 1 -a $# -gt 1 ]; then - grep_args="-H ${grep_args}" + if [ "${silent}" -lt 1 ] && [ "$#" -gt 1 ]; then + grep_args="-H ${grep_args}" fi - while [ $# -gt 0 ] - do - ${zcat} -fq -- "$1" | ${grep} --label="${1}" ${grep_args} -- "${pattern}" - - shift + CUR_EXIT_CODE=0 + EXIT_CODE=1 + while [ "$#" -gt 0 ]; do + # shellcheck disable=SC2086 + "${zcat}" -fq -- "$1" | "${grep}" --label="${1}" ${grep_args} -- "${pattern}" - + CUR_EXIT_CODE=$? + if [ "${CUR_EXIT_CODE}" -eq 0 ] && [ "${EXIT_CODE}" -ne 1 ]; then + EXIT_CODE=0 + fi + shift done fi -exit 0 +exit "${EXIT_CODE}" From 3d18b4764dd3c839361c70aded843d9984410011 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 03:04:40 +0700 Subject: [PATCH 08/62] Prevent globbing on non-quoting variable --- programs/zstdgrep | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/programs/zstdgrep b/programs/zstdgrep index 7f050d04a..a10e0710a 100755 --- a/programs/zstdgrep +++ b/programs/zstdgrep @@ -103,9 +103,11 @@ EXIT_CODE=0 # call grep ... if [ "$#" -lt 1 ]; then # ... on stdin + set -f # Disable file name generation (globbing). # shellcheck disable=SC2086 "${zcat}" -fq - | "${grep}" ${grep_args} -- "${pattern}" - EXIT_CODE=$? + set +f else # ... on all files given on the command line if [ "${silent}" -lt 1 ] && [ "$#" -gt 1 ]; then @@ -113,6 +115,7 @@ else fi CUR_EXIT_CODE=0 EXIT_CODE=1 + set -f while [ "$#" -gt 0 ]; do # shellcheck disable=SC2086 "${zcat}" -fq -- "$1" | "${grep}" --label="${1}" ${grep_args} -- "${pattern}" - @@ -122,6 +125,7 @@ else fi shift done + set +f fi exit "${EXIT_CODE}" From d095adf9fb7fcadfaa770e2c75d14dde0950f6d0 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 03:39:47 +0700 Subject: [PATCH 09/62] Add simple test for zstdgrep --- tests/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index f363001b1..25bd5c84e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -320,7 +320,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 test-all: test test32 valgrindTest test-decodecorpus-cli -.PHONY: test-zstd test-zstd32 test-zstd-nolegacy +.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep test-zstd: ZSTD = $(PRGDIR)/zstd test-zstd: zstd @@ -352,6 +352,10 @@ test-gzstd: gzstd $(PRGDIR)/zstd -dcf - Date: Tue, 20 Nov 2018 12:37:43 -0800 Subject: [PATCH 10/62] [regression] Add initial regression test framework The regression tests run nightly or on the `regression` branch for convenience. The results get uploaded as the artifacts of the job. If they change, check the diff printed in the job. If all is well, download the new results and commit them to the repo. This code will only run on a UNIX like platform. It could be made to run on Windows, but I don't think that it is necessary. It also uses C99. * data: This module defines the data to run tests on. It downloads data from a URL into a cache directory, checks it against a checksum, and unpacks it. It also provides helpers for accessing the data. * config: This module defines the configs to run tests with. A config is a set of API parameters and a set of CLI flags. * result: This module is a helper for method that defines the result type. * method: This module defines the compression methods to test. It is what runs the regression test using the data and the config. It reports the total compressed size, or an error/skip. * test: This is the test binary that runs the tests for every (data, config, method) tuple, and prints the results to the output file and stderr. * results.csv: The results that the current commit is expected to produce. --- .circleci/config.yml | 67 ++++- tests/regression/Makefile | 58 +++++ tests/regression/config.c | 63 +++++ tests/regression/config.h | 59 +++++ tests/regression/data.c | 493 +++++++++++++++++++++++++++++++++++ tests/regression/data.h | 92 +++++++ tests/regression/levels.h | 44 ++++ tests/regression/method.c | 186 +++++++++++++ tests/regression/method.h | 65 +++++ tests/regression/result.c | 28 ++ tests/regression/result.h | 103 ++++++++ tests/regression/results.csv | 43 +++ tests/regression/test.c | 308 ++++++++++++++++++++++ 13 files changed, 1608 insertions(+), 1 deletion(-) create mode 100644 tests/regression/Makefile create mode 100644 tests/regression/config.c create mode 100644 tests/regression/config.h create mode 100644 tests/regression/data.c create mode 100644 tests/regression/data.h create mode 100644 tests/regression/levels.h create mode 100644 tests/regression/method.c create mode 100644 tests/regression/method.h create mode 100644 tests/regression/result.c create mode 100644 tests/regression/result.h create mode 100644 tests/regression/results.csv create mode 100644 tests/regression/test.c diff --git a/.circleci/config.yml b/.circleci/config.yml index b08634408..42e4042db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,8 @@ references: sudo apt-get -y install \ gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \ libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \ - libc6-dev-ppc64-powerpc-cross zstd gzip coreutils + libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \ + libcurl4-openssl-dev jobs: # the first half of the jobs are in this test @@ -82,6 +83,49 @@ jobs: cp $ZSTD_VERSION.tar* $CIRCLE_ARTIFACTS - store_artifacts: path: /tmp/circleci-artifacts + # This step should only be run in a cron job + regression-test: + docker: + - image: circleci/buildpack-deps:bionic + environment: + CIRCLE_ARTIFACTS: /tmp/circleci-artifacts + steps: + - checkout + - *install-dependencies + # Restore the cached resources. + - restore_cache: + # We try our best to bust the cache when the data changes by hashing + # data.c. If that doesn't work, simply update the version number here + # and below. If we fail to bust the cache, the regression testing will + # still work, since it has its own stamp, but will need to redownload + # everything. + keys: + - regression-cache-{{ checksum "tests/regression/data.c" }}-v0 + - run: + name: Regression Test + command: | + make -C programs zstd + make -C tests/regression test + mkdir -p $CIRCLE_ARTIFACTS + ./tests/regression/test \ + --cache tests/regression/cache \ + --output $CIRCLE_ARTIFACTS/results.csv \ + --zstd programs/zstd + echo "NOTE: The new results.csv is uploaded as an artifact to this job" + echo " If this fails, go to the Artifacts pane in CircleCI, " + echo " download /tmp/circleci-artifacts/results.csv, and if they " + echo " are still good, copy it into the repo and commit it." + echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv" + diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv + # Only save the cache on success (default), since if the failure happened + # before we stamp the data cache, we will have a bad cache for this key. + - save_cache: + key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0 + paths: + - tests/regression/cache + - store_artifacts: + path: /tmp/circleci-artifacts + workflows: version: 2 @@ -96,6 +140,13 @@ workflows: filters: tags: only: /.*/ + # Create a branch called regression and set it to dev to force a + # regression test run + - regression-test: + filters: + branches: + only: + - regression # Only run on release tags. - publish-github-release: requires: @@ -106,6 +157,20 @@ workflows: ignore: /.*/ tags: only: /^v\d+\.\d+\.\d+$/ + nightly: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - master + - dev + jobs: + # Run daily long regression tests + - regression-test + + # Longer tests #- make -C tests test-zstd-nolegacy && make clean diff --git a/tests/regression/Makefile b/tests/regression/Makefile new file mode 100644 index 000000000..2008b2f7e --- /dev/null +++ b/tests/regression/Makefile @@ -0,0 +1,58 @@ +# ################################################################ +# Copyright (c) 2015-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + +CFLAGS ?= -O3 + +CURL_CFLAGS := $(shell curl-config --cflags) +CURL_LDFLAGS := $(shell curl-config --libs) + +PROGDIR := ../../programs +LIBDIR := ../../lib +ZSTD_CPPFLAGS := -I$(PROGDIR) -I$(LIBDIR) -I$(LIBDIR)/common + +REGRESSION_CFLAGS = $(CFLAGS) $(CURL_CFLAGS) +REGRESSION_CPPFLAGS = $(CPPFLAGS) $(ZSTD_CPPFLAGS) +REGRESSION_LDFLAGS = $(LDFLAGS) $(CURL_LDFLAGS) + +all: test + +xxhash.o: $(LIBDIR)/common/xxhash.c $(LIBDIR)/common/xxhash.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +util.o: $(PROGDIR)/util.c $(PROGDIR)/util.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +data.o: data.c data.h $(PROGDIR)/util.h $(LIBDIR)/common/xxhash.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +config.o: config.c config.h levels.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +method.h: data.h config.h result.h + +method.o: method.c method.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +result.o: result.c result.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +test.o: test.c data.h config.h method.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +libzstd.a: + $(MAKE) -C $(LIBDIR) libzstd.a + cp $(LIBDIR)/libzstd.a . + +test: test.o data.o config.o util.o method.o result.o xxhash.o libzstd.a + $(CC) $^ $(REGRESSION_LDFLAGS) -o $@ + +.PHONY: clean +clean: + $(MAKE) -C $(LIBDIR) clean + $(RM) *.o *.a test diff --git a/tests/regression/config.c b/tests/regression/config.c new file mode 100644 index 000000000..38ac0091b --- /dev/null +++ b/tests/regression/config.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "config.h" + +/* Define a config for each fast level we want to test with. */ +#define FAST_LEVEL(x) \ + param_value_t const level_fast##x##_param_values[] = { \ + {.param = ZSTD_p_compressionLevel, .value = (unsigned)-x}, \ + }; \ + config_t const level_fast##x = { \ + .name = "level -" #x, \ + .cli_args = "--fast=" #x, \ + .param_values = PARAM_VALUES(level_fast##x##_param_values), \ + }; + +/* Define a config for each level we want to test with. */ +#define LEVEL(x) \ + param_value_t const level_##x##_param_values[] = { \ + {.param = ZSTD_p_compressionLevel, .value = (unsigned)x}, \ + }; \ + config_t const level_##x = { \ + .name = "level " #x, \ + .cli_args = "-" #x, \ + .param_values = PARAM_VALUES(level_##x##_param_values), \ + }; + + +#define PARAM_VALUES(pv) \ + { .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) } + +#include "levels.h" + +#undef LEVEL +#undef FAST_LEVEL + +static config_t const* g_configs[] = { +#define FAST_LEVEL(x) &level_fast##x, +#define LEVEL(x) &level_##x, +#include "levels.h" +#undef LEVEL +#undef FAST_LEVEL + NULL, +}; + +config_t const* const* configs = g_configs; + +int config_get_level(config_t const* config) { + param_values_t const params = config->param_values; + size_t i; + for (size_t i = 0; i < params.size; ++i) { + if (params.data[i].param == ZSTD_p_compressionLevel) + return params.data[i].value; + } + return CONFIG_NO_LEVEL; +} diff --git a/tests/regression/config.h b/tests/regression/config.h new file mode 100644 index 000000000..73f1e9f8a --- /dev/null +++ b/tests/regression/config.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef CONFIG_H +#define CONFIG_H + +#include + +#define ZSTD_STATIC_LINKING_ONLY +#include + +typedef struct { + ZSTD_cParameter param; + unsigned value; +} param_value_t; + +typedef struct { + size_t size; + param_value_t const* data; +} param_values_t; + +/** + * The config tells the compression method what options to use. + */ +typedef struct { + const char* name; /**< Identifies the config in the results table */ + /** + * Optional arguments to pass to the CLI. If not set, CLI-based methods + * will skip this config. + */ + char const* cli_args; + /** + * Parameters to pass to the advanced API. If the advanced API isn't used, + * the parameters will be derived from these. + */ + param_values_t param_values; +} config_t; + +#define CONFIG_NO_LEVEL (-ZSTD_TARGETLENGTH_MAX - 1) +/** + * Returns the compression level specified by the config, or CONFIG_NO_LEVEL if + * no level is specified. Note that 0 is a valid compression level, meaning + * default. + */ +int config_get_level(config_t const* config); + +/** + * The NULL-terminated list of configs. + */ +extern config_t const* const* configs; + +#endif diff --git a/tests/regression/data.c b/tests/regression/data.c new file mode 100644 index 000000000..68269e042 --- /dev/null +++ b/tests/regression/data.c @@ -0,0 +1,493 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "data.h" + +#include +#include +#include +#include + +#include + +#include + +#include "mem.h" +#include "util.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + +/** + * Data objects + */ + +#define REGRESSION_RELEASE(x) \ + "https://github.com/facebook/zstd/releases/download/regression-data/" x + +data_t silesia = { + .url = REGRESSION_RELEASE("silesia.tar.zst"), + .name = "silesia", + .type = data_type_dir, + .xxhash64 = 0x67558ee5506918b4LL, +}; + +data_t silesia_tar = { + .url = REGRESSION_RELEASE("silesia.tar.zst"), + .name = "silesia.tar", + .type = data_type_file, + .xxhash64 = 0x67558ee5506918b4LL, +}; + +static data_t* g_data[] = { + &silesia, + &silesia_tar, + NULL, +}; + +data_t const* const* data = (data_t const* const*)g_data; + +/** + * data buffer helper functions (documented in header). + */ + +data_buffer_t data_buffer_create(size_t const capacity) { + data_buffer_t buffer = {}; + + buffer.data = (uint8_t*)malloc(capacity); + if (buffer.data == NULL) + return buffer; + buffer.capacity = capacity; + return buffer; +} + +data_buffer_t data_buffer_read(char const* filename) { + data_buffer_t buffer = {}; + + uint64_t const size = UTIL_getFileSize(filename); + if (size == UTIL_FILESIZE_UNKNOWN) { + fprintf(stderr, "unknown size for %s\n", filename); + return buffer; + } + + buffer.data = (uint8_t*)malloc(size); + if (buffer.data == NULL) { + fprintf(stderr, "malloc failed\n"); + return buffer; + } + buffer.capacity = size; + + FILE* file = fopen(filename, "rb"); + if (file == NULL) { + fprintf(stderr, "file null\n"); + goto err; + } + buffer.size = fread(buffer.data, 1, buffer.capacity, file); + fclose(file); + if (buffer.size != buffer.capacity) { + fprintf(stderr, "read %zu != %zu\n", buffer.size, buffer.capacity); + goto err; + } + + return buffer; +err: + free(buffer.data); + memset(&buffer, 0, sizeof(buffer)); + return buffer; + +} + +data_buffer_t data_buffer_get(data_t const* data) { + data_buffer_t const kEmptyBuffer = {}; + + if (data->type != data_type_file) + return kEmptyBuffer; + + return data_buffer_read(data->path); +} + +int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2) { + size_t const size = + buffer1.size < buffer2.size ? buffer1.size : buffer2.size; + int const cmp = memcmp(buffer1.data, buffer2.data, size); + if (cmp != 0) + return cmp; + if (buffer1.size < buffer2.size) + return -1; + if (buffer1.size == buffer2.size) + return 0; + assert(buffer1.size > buffer2.size); + return 1; + +} + +void data_buffer_free(data_buffer_t buffer) { + free(buffer.data); +} + +/** + * Initialization and download functions. + */ + +static char* g_data_dir = NULL; + +/* mkdir -p */ +static int ensure_directory_exists(char const* indir) { + char* const dir = strdup(indir); + char* end = dir; + int ret = 0; + if (dir == NULL) { + ret = EINVAL; + goto out; + } + do { + /* Find the next directory level. */ + for (++end; *end != '\0' && *end != '/'; ++end) + ; + /* End the string there, make the directory, and restore the string. */ + char const save = *end; + *end = '\0'; + int const isdir = UTIL_isDirectory(dir); + ret = mkdir(dir, S_IRWXU); + *end = save; + /* Its okay if the directory already exists. */ + if (ret == 0 || (errno == EEXIST && isdir)) + continue; + ret = errno; + fprintf(stderr, "mkdir() failed\n"); + goto out; + } while (*end != '\0'); + + ret = 0; +out: + free(dir); + return ret; +} + +/** Concatenate 3 strings into a new buffer. */ +static char* cat3(char const* str1, char const* str2, char const* str3) { + size_t const size1 = strlen(str1); + size_t const size2 = strlen(str2); + size_t const size3 = strlen(str3); + size_t const size = size1 + size2 + size3 + 1; + char* const dst = (char*)malloc(size); + if (dst == NULL) + return NULL; + strcpy(dst, str1); + strcpy(dst + size1, str2); + strcpy(dst + size1 + size2, str3); + assert(strlen(dst) == size1 + size2 + size3); + return dst; +} + +/** + * State needed by the curl callback. + * It takes data from curl, hashes it, and writes it to the file. + */ +typedef struct { + FILE* file; + XXH64_state_t xxhash64; + int error; +} curl_data_t; + +/** Create the curl state. */ +static curl_data_t curl_data_create(data_t const* data) { + curl_data_t cdata = {}; + + XXH64_reset(&cdata.xxhash64, 0); + + assert(UTIL_isDirectory(g_data_dir)); + + if (data->type == data_type_file) { + /* Decompress the resource and store to the path. */ + char* cmd = cat3("zstd -dqfo '", data->path, "'"); + if (cmd == NULL) { + cdata.error = ENOMEM; + return cdata; + } + cdata.file = popen(cmd, "w"); + free(cmd); + } else { + /* Decompress and extract the resource to the cache directory. */ + char* cmd = cat3("zstd -dc | tar -x -C '", g_data_dir, "'"); + if (cmd == NULL) { + cdata.error = ENOMEM; + return cdata; + } + cdata.file = popen(cmd, "w"); + free(cmd); + } + if (cdata.file == NULL) { + cdata.error = errno; + } + + return cdata; +} + +/** Free the curl state. */ +static int curl_data_free(curl_data_t cdata) { + return pclose(cdata.file); +} + +/** curl callback. Updates the hash, and writes to the file. */ +static size_t curl_write(void* data, size_t size, size_t count, void* ptr) { + curl_data_t* cdata = (curl_data_t*)ptr; + size_t const written = fwrite(data, size, count, cdata->file); + XXH64_update(&cdata->xxhash64, data, written * size); + return written; +} + +/** Download a single data object. */ +static int curl_download_datum(CURL* curl, data_t const* data) { + curl_data_t cdata = curl_data_create(data); + int err = EFAULT; + + if (cdata.error != 0) { + err = cdata.error; + goto out; + } + + /* Download the data. */ + if (curl_easy_setopt(curl, CURLOPT_URL, data->url) != 0) + goto out; + if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cdata) != 0) + goto out; + if (curl_easy_perform(curl) != 0) { + fprintf(stderr, "downloading '%s' failed\n", data->url); + goto out; + } + /* check that the file exists. */ + if (data->type == data_type_file && !UTIL_isRegularFile(data->path)) { + fprintf(stderr, "output file '%s' does not exist\n", data->path); + goto out; + } + if (data->type == data_type_dir && !UTIL_isDirectory(data->path)) { + fprintf(stderr, "output directory '%s' does not exist\n", data->path); + goto out; + } + /* Check that the hash matches. */ + if (XXH64_digest(&cdata.xxhash64) != data->xxhash64) { + fprintf( + stderr, + "checksum does not match: %llx != %llx\n", + (unsigned long long)XXH64_digest(&cdata.xxhash64), + (unsigned long long)data->xxhash64); + goto out; + } + + err = 0; +out: + if (err != 0) + fprintf(stderr, "downloading '%s' failed\n", data->name); + int const close_err = curl_data_free(cdata); + if (close_err != 0 && err == 0) { + fprintf(stderr, "failed to write data for '%s'\n", data->name); + err = close_err; + } + return err; +} + +/** Download all the data. */ +static int curl_download_data(data_t const* const* data) { + if (curl_global_init(CURL_GLOBAL_ALL) != 0) + return EFAULT; + + curl_data_t cdata = {}; + CURL* curl = curl_easy_init(); + int err = EFAULT; + + if (curl == NULL) + return EFAULT; + + if (curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L) != 0) + goto out; + if (curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L) != 0) + goto out; + if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write) != 0) + goto out; + + assert(data != NULL); + for (; *data != NULL; ++data) { + if (curl_download_datum(curl, *data) != 0) + goto out; + } + + err = 0; +out: + curl_easy_cleanup(curl); + curl_global_cleanup(); + return err; +} + +/** Fill the path member variable of the data objects. */ +static int data_create_paths(data_t* const* data, char const* dir) { + size_t const dirlen = strlen(dir); + assert(data != NULL); + for (; *data != NULL; ++data) { + data_t* const datum = *data; + datum->path = cat3(dir, "/", datum->name); + if (datum->path == NULL) + return ENOMEM; + } + return 0; +} + +/** Free the path member variable of the data objects. */ +static void data_free_paths(data_t* const* data) { + assert(data != NULL); + for (; *data != NULL; ++data) { + data_t* datum = *data; + free((void*)datum->path); + datum->path = NULL; + } +} + +static char const kStampName[] = "STAMP"; + +static void xxh_update_le(XXH64_state_t* state, uint64_t data) { + if (!MEM_isLittleEndian()) + data = MEM_swap64(data); + XXH64_update(state, &data, sizeof(data)); +} + +/** Hash the data to create the stamp. */ +static uint64_t stamp_hash(data_t const* const* data) { + XXH64_state_t state; + + XXH64_reset(&state, 0); + assert(data != NULL); + for (; *data != NULL; ++data) { + data_t const* datum = *data; + /* We don't care about the URL that we fetch from. */ + /* The path is derived from the name. */ + XXH64_update(&state, datum->name, strlen(datum->name)); + xxh_update_le(&state, datum->xxhash64); + xxh_update_le(&state, datum->type); + } + return XXH64_digest(&state); +} + +/** Check if the stamp matches the stamp in the cache directory. */ +static int stamp_check(char const* dir, data_t const* const* data) { + char* stamp = cat3(dir, "/", kStampName); + uint64_t const expected = stamp_hash(data); + XXH64_canonical_t actual; + FILE* stampfile = NULL; + int matches = 0; + + if (stamp == NULL) + goto out; + if (!UTIL_isRegularFile(stamp)) { + fprintf(stderr, "stamp does not exist: recreating the data cache\n"); + goto out; + } + + stampfile = fopen(stamp, "rb"); + if (stampfile == NULL) { + fprintf(stderr, "could not open stamp: recreating the data cache\n"); + goto out; + } + + size_t b; + if ((b = fread(&actual, sizeof(actual), 1, stampfile)) != 1) { + fprintf(stderr, "invalid stamp: recreating the data cache\n"); + goto out; + } + + matches = (expected == XXH64_hashFromCanonical(&actual)); + if (matches) + fprintf(stderr, "stamp matches: reusing the cached data\n"); + else + fprintf(stderr, "stamp does not match: recreating the data cache\n"); + +out: + free(stamp); + if (stampfile != NULL) + fclose(stampfile); + return matches; +} + +/** On success write a new stamp, on failure delete the old stamp. */ +static int +stamp_write(char const* dir, data_t const* const* data, int const data_err) { + char* stamp = cat3(dir, "/", kStampName); + FILE* stampfile = NULL; + int err = EIO; + + if (stamp == NULL) + return ENOMEM; + + if (data_err != 0) { + err = data_err; + goto out; + } + XXH64_canonical_t hash; + + XXH64_canonicalFromHash(&hash, stamp_hash(data)); + + stampfile = fopen(stamp, "wb"); + if (stampfile == NULL) + goto out; + if (fwrite(&hash, sizeof(hash), 1, stampfile) != 1) + goto out; + err = 0; + fprintf(stderr, "stamped new data cache\n"); +out: + if (err != 0) + /* Ignore errors. */ + unlink(stamp); + free(stamp); + if (stampfile != NULL) + fclose(stampfile); + return err; +} + +int data_init(char const* dir) { + int err; + + if (dir == NULL) + return EINVAL; + + /* This must be first to simplify logic. */ + err = ensure_directory_exists(dir); + if (err != 0) + return err; + + /* Save the cache directory. */ + g_data_dir = strdup(dir); + if (g_data_dir == NULL) + return ENOMEM; + + err = data_create_paths(g_data, dir); + if (err != 0) + return err; + + /* If the stamp matches then we are good to go. + * This must be called before any modifications to the data cache. + * After this point, we MUST call stamp_write() to update the STAMP, + * since we've updated the data cache. + */ + if (stamp_check(dir, data)) + return 0; + + err = curl_download_data(data); + if (err != 0) + goto out; + +out: + /* This must be last, since it must know if data_init() succeeded. */ + stamp_write(dir, data, err); + return err; +} + +void data_finish(void) { + data_free_paths(g_data); + free(g_data_dir); + g_data_dir = NULL; +} diff --git a/tests/regression/data.h b/tests/regression/data.h new file mode 100644 index 000000000..d75f645b1 --- /dev/null +++ b/tests/regression/data.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef DATA_H +#define DATA_H + +#include +#include + +typedef enum { + data_type_file = 1, /**< This data is a file. *.zst */ + data_type_dir = 2, /**< This data is a directory. *.tar.zst */ +} data_type_t; + +typedef struct { + char const* url; /**< Where to get this resource. */ + uint64_t xxhash64; /**< Hash of the url contents. */ + char const* name; /**< The logical name of the resource (no extension). */ + data_type_t type; /**< The type of this resource. */ + char const* path; /**< The path of the unpacked resource (derived). */ + size_t size; +} data_t; + +/** + * The NULL-terminated list of data objects. + */ +extern data_t const* const* data; + +/** + * Initializes the data module and downloads the data necessary. + * Caches the downloads in dir. We add a stamp file in the directory after + * a successful download. If a stamp file already exists, and matches our + * current data stamp, we will use the cached data without downloading. + * + * @param dir The directory to cache the downloaded data into. + * + * @returns 0 on success. + */ +int data_init(char const* dir); + +/** + * Must be called at exit to free resources allocated by data_init(). + */ +void data_finish(void); + +typedef struct { + uint8_t* data; + size_t size; + size_t capacity; +} data_buffer_t; + +/** + * Read the file that data points to into a buffer. + * NOTE: data must be a file, not a directory. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_get(data_t const* data); + +/** + * Read the contents of filename into a buffer. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_read(char const* filename); + +/** + * Create a buffer with the specified capacity. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_create(size_t capacity); + +/** + * Calls memcmp() on the contents [0, size) of both buffers. + */ +int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2); + +/** + * Frees an allocated buffer. + */ +void data_buffer_free(data_buffer_t buffer); + + +#endif diff --git a/tests/regression/levels.h b/tests/regression/levels.h new file mode 100644 index 000000000..f96689075 --- /dev/null +++ b/tests/regression/levels.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef LEVEL +# error LEVEL(x) must be defined +#endif +#ifndef FAST_LEVEL +# error FAST_LEVEL(x) must be defined +#endif + +/** + * The levels are chosen to trigger every strategy in every source size, + * as well as some fast levels and the default level. + * If you change the compression levels, you should probably update these. + */ + +FAST_LEVEL(5) + +FAST_LEVEL(3) + +FAST_LEVEL(1) +LEVEL(0) +LEVEL(1) + +LEVEL(3) +LEVEL(4) +LEVEL(5) +LEVEL(6) +LEVEL(7) + +LEVEL(9) + +LEVEL(13) + +LEVEL(16) + +LEVEL(19) diff --git a/tests/regression/method.c b/tests/regression/method.c new file mode 100644 index 000000000..c19a377a7 --- /dev/null +++ b/tests/regression/method.c @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "method.h" + +#include +#include + +#include + +static char const* g_zstdcli = NULL; + +void method_set_zstdcli(char const* zstdcli) { + g_zstdcli = zstdcli; +} + +/** + * Macro to get a pointer of type, given ptr, which is a member variable with + * the given name, member. + * + * method_state_t* base = ...; + * simple_state_t* state = container_of(base, simple_state_t, base); + */ +#define container_of(ptr, type, member) \ + ((type*)(char*)(ptr)-offsetof(type, member)) + +/** State to reuse the same buffers between compression calls. */ +typedef struct { + method_state_t base; + data_buffer_t buffer; /**< The constant input data buffer. */ + data_buffer_t compressed; /**< The compressed data buffer. */ + data_buffer_t decompressed; /**< The decompressed data buffer. */ +} simple_state_t; + +static method_state_t* simple_create(data_t const* data) { + simple_state_t* state = (simple_state_t*)calloc(1, sizeof(simple_state_t)); + if (state == NULL) + return NULL; + state->base.data = data; + state->buffer = data_buffer_get(data); + state->compressed = + data_buffer_create(ZSTD_compressBound(state->buffer.size)); + state->decompressed = data_buffer_create(state->buffer.size); + return &state->base; +} + +static void simple_destroy(method_state_t* base) { + if (base == NULL) + return; + simple_state_t* state = container_of(base, simple_state_t, base); + free(state); +} + +static result_t simple_compress(method_state_t* base, config_t const* config) { + if (base == NULL) + return result_error(result_error_system_error); + simple_state_t* state = container_of(base, simple_state_t, base); + + if (base->data->type != data_type_file) + return result_error(result_error_skip); + + if (state->buffer.data == NULL || state->compressed.data == NULL || + state->decompressed.data == NULL) { + return result_error(result_error_system_error); + } + + /* If the config doesn't specify a level, skip. */ + int const level = config_get_level(config); + if (level == CONFIG_NO_LEVEL) + return result_error(result_error_skip); + + /* Compress, decompress, and check the result. */ + state->compressed.size = ZSTD_compress( + state->compressed.data, + state->compressed.capacity, + state->buffer.data, + state->buffer.size, + level); + if (ZSTD_isError(state->compressed.size)) + return result_error(result_error_compression_error); + + state->decompressed.size = ZSTD_decompress( + state->decompressed.data, + state->decompressed.capacity, + state->compressed.data, + state->compressed.size); + if (ZSTD_isError(state->decompressed.size)) + return result_error(result_error_decompression_error); + if (data_buffer_compare(state->buffer, state->decompressed)) + return result_error(result_error_round_trip_error); + + result_data_t data; + data.total_size = state->compressed.size; + return result_data(data); +} + +/** Generic state creation function. */ +static method_state_t* method_state_create(data_t const* data) { + method_state_t* state = (method_state_t*)malloc(sizeof(method_state_t)); + if (state == NULL) + return NULL; + state->data = data; + return state; +} + +static void method_state_destroy(method_state_t* state) { + free(state); +} + +#define MAX_OUT 32 + +static result_t cli_file_compress( + method_state_t* state, + config_t const* config) { + if (config->cli_args == NULL) + return result_error(result_error_skip); + + if (g_zstdcli == NULL) + return result_error(result_error_system_error); + + /* '' -r '' | wc -c */ + char cmd[1024]; + size_t const cmd_size = snprintf( + cmd, + sizeof(cmd), + "'%s' -cqr %s '%s' | wc -c", + g_zstdcli, + config->cli_args, + state->data->path); + if (cmd_size >= sizeof(cmd)) { + fprintf(stderr, "command too large: %s\n", cmd); + return result_error(result_error_system_error); + } + FILE* zstd = popen(cmd, "r"); + if (zstd == NULL) { + fprintf(stderr, "failed to popen command: %s\n", cmd); + return result_error(result_error_system_error); + } + + /* Read the total compressed size. */ + char out[MAX_OUT + 1]; + size_t const out_size = fread(out, 1, MAX_OUT, zstd); + out[out_size] = '\0'; + int const zstd_ret = pclose(zstd); + if (zstd_ret != 0) { + fprintf(stderr, "zstd failed with command: %s\n", cmd); + return result_error(result_error_compression_error); + } + if (out_size == MAX_OUT) { + fprintf(stderr, "wc -c produced more bytes than expected: %s\n", out); + return result_error(result_error_system_error); + } + + result_data_t data; + data.total_size = atoll(out); + return result_data(data); +} + +method_t const simple = { + .name = "simple", + .create = simple_create, + .compress = simple_compress, + .destroy = simple_destroy, +}; + +method_t const cli_file = { + .name = "cli file", + .create = method_state_create, + .compress = cli_file_compress, + .destroy = method_state_destroy, +}; + +static method_t const* g_methods[] = { + &simple, + &cli_file, + NULL, +}; + +method_t const* const* methods = g_methods; diff --git a/tests/regression/method.h b/tests/regression/method.h new file mode 100644 index 000000000..d70b776b1 --- /dev/null +++ b/tests/regression/method.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef METHOD_H +#define METHOD_H + +#include + +#include "data.h" +#include "config.h" +#include "result.h" + +/** + * The base class for state that methods keep. + * All derived method state classes must have a member of this type. + */ +typedef struct { + data_t const* data; +} method_state_t; + +/** + * A method that compresses the data using config. + */ +typedef struct { + char const* name; /**< The identifier for this method in the results. */ + /** + * Creates a state that must contain a member variable of method_state_t, + * and returns a pointer to that member variable. + * + * This method can be used to do expensive work that only depends on the + * data, like loading the data file into a buffer. + */ + method_state_t* (*create)(data_t const* data); + /** + * Compresses the data in the state using the given config. + * + * @param state A pointer to the state returned by create(). + * + * @returns The total compressed size on success, or an error code. + */ + result_t (*compress)(method_state_t* state, config_t const* config); + /** + * Frees the state. + */ + void (*destroy)(method_state_t* state); +} method_t; + +/** + * Set the zstd cli path. Must be called before any methods are used. + */ +void method_set_zstdcli(char const* zstdcli); + +/** + * A NULL-terminated list of methods. + */ +extern method_t const* const* methods; + +#endif diff --git a/tests/regression/result.c b/tests/regression/result.c new file mode 100644 index 000000000..31439b08c --- /dev/null +++ b/tests/regression/result.c @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "result.h" + +char const* result_get_error_string(result_t result) { + switch (result_get_error(result)) { + case result_error_ok: + return "okay"; + case result_error_skip: + return "skip"; + case result_error_system_error: + return "system error"; + case result_error_compression_error: + return "compression error"; + case result_error_decompression_error: + return "decompression error"; + case result_error_round_trip_error: + return "round trip error"; + } +} diff --git a/tests/regression/result.h b/tests/regression/result.h new file mode 100644 index 000000000..8c80cf85a --- /dev/null +++ b/tests/regression/result.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef RESULT_H +#define RESULT_H + +#include + +/** + * The error type enum. + */ +typedef enum { + result_error_ok, /**< No error. */ + result_error_skip, /**< This method was skipped. */ + result_error_system_error, /**< Some internal error happened. */ + result_error_compression_error, /**< Compression failed. */ + result_error_decompression_error, /**< Decompression failed. */ + result_error_round_trip_error, /**< Data failed to round trip. */ +} result_error_t; + +/** + * The success type. + */ +typedef struct { + size_t total_size; /**< The total compressed size. */ +} result_data_t; + +/** + * The result type. + * Do not access the member variables directory, use the helper functions. + */ +typedef struct { + result_error_t internal_error; + result_data_t internal_data; +} result_t; + +/** + * Create a result of the error type. + */ +static result_t result_error(result_error_t error); +/** + * Create a result of the success type. + */ +static result_t result_data(result_data_t data); + +/** + * Check if the result is an error or skip. + */ +static int result_is_error(result_t result); +/** + * Check if the result error is skip. + */ +static int result_is_skip(result_t result); +/** + * Get the result error or okay. + */ +static result_error_t result_get_error(result_t result); +/** + * Get the result data. The result MUST be checked with result_is_error() first. + */ +static result_data_t result_get_data(result_t result); + +static result_t result_error(result_error_t error) { + result_t result = { + .internal_error = error, + }; + return result; +} + +static result_t result_data(result_data_t data) { + result_t result = { + .internal_error = result_error_ok, + .internal_data = data, + }; + return result; +} + +static int result_is_error(result_t result) { + return result_get_error(result) != result_error_ok; +} + +static int result_is_skip(result_t result) { + return result_get_error(result) == result_error_skip; +} + +static result_error_t result_get_error(result_t result) { + return result.internal_error; +} + +char const* result_get_error_string(result_t result); + +static result_data_t result_get_data(result_t result) { + return result.internal_data; +} + +#endif diff --git a/tests/regression/results.csv b/tests/regression/results.csv new file mode 100644 index 000000000..8ed689d26 --- /dev/null +++ b/tests/regression/results.csv @@ -0,0 +1,43 @@ +Data, Config, Method, Total compressed size +silesia.tar, level -5, simple, 106176430 +silesia.tar, level -3, simple, 98476550 +silesia.tar, level -1, simple, 87206767 +silesia.tar, level 0, simple, 66996953 +silesia.tar, level 1, simple, 73658303 +silesia.tar, level 3, simple, 66996953 +silesia.tar, level 4, simple, 65996020 +silesia.tar, level 5, simple, 64421326 +silesia.tar, level 6, simple, 62388673 +silesia.tar, level 7, simple, 61159525 +silesia.tar, level 9, simple, 60214921 +silesia.tar, level 13, simple, 58428642 +silesia.tar, level 16, simple, 56363759 +silesia.tar, level 19, simple, 53274173 +silesia, level -5, cli file, 106202112 +silesia, level -3, cli file, 98518660 +silesia, level -1, cli file, 87226203 +silesia, level 0, cli file, 67049190 +silesia, level 1, cli file, 73676282 +silesia, level 3, cli file, 67049190 +silesia, level 4, cli file, 66090040 +silesia, level 5, cli file, 64503721 +silesia, level 6, cli file, 62446177 +silesia, level 7, cli file, 61217029 +silesia, level 9, cli file, 60282841 +silesia, level 13, cli file, 58480658 +silesia, level 16, cli file, 56414170 +silesia, level 19, cli file, 53365292 +silesia.tar, level -5, cli file, 106250113 +silesia.tar, level -3, cli file, 98550747 +silesia.tar, level -1, cli file, 87227322 +silesia.tar, level 0, cli file, 67111168 +silesia.tar, level 1, cli file, 73694374 +silesia.tar, level 3, cli file, 67111168 +silesia.tar, level 4, cli file, 66154079 +silesia.tar, level 5, cli file, 64546998 +silesia.tar, level 6, cli file, 62458454 +silesia.tar, level 7, cli file, 61231085 +silesia.tar, level 9, cli file, 60310313 +silesia.tar, level 13, cli file, 58517476 +silesia.tar, level 16, cli file, 56448694 +silesia.tar, level 19, cli file, 53444920 diff --git a/tests/regression/test.c b/tests/regression/test.c new file mode 100644 index 000000000..c3063511c --- /dev/null +++ b/tests/regression/test.c @@ -0,0 +1,308 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include +#include +#include +#include + +#include "config.h" +#include "data.h" +#include "method.h" + +/** Check if a name contains a comma. */ +static int is_name_bad(char const* name) { + if (name == NULL) + return 1; + for (; *name != '\0'; ++name) + if (*name == ',') + return 1; + return 0; +} + +/** Check if any of the names contain a comma. */ +static int are_names_bad() { + for (size_t method = 0; methods[method] != NULL; ++method) + if (is_name_bad(methods[method]->name)) { + fprintf(stderr, "method name %s is bad\n", methods[method]->name); + return 1; + } + for (size_t datum = 0; data[datum] != NULL; ++datum) + if (is_name_bad(data[datum]->name)) { + fprintf(stderr, "data name %s is bad\n", data[datum]->name); + return 1; + } + for (size_t config = 0; configs[config] != NULL; ++config) + if (is_name_bad(configs[config]->name)) { + fprintf(stderr, "config name %s is bad\n", configs[config]->name); + return 1; + } + return 0; +} + +/** Helper macro to print to stderr and a file. */ +#define tprintf(file, ...) \ + do { \ + fprintf(file, __VA_ARGS__); \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) +/** Helper macro to flush stderr and a file. */ +#define tflush(file) \ + do { \ + fflush(file); \ + fflush(stderr); \ + } while (0) + +/** + * Run all the regression tests and record the results table to results and + * stderr progressively. + */ +static int run_all(FILE* results) { + tprintf(results, "Data,\tConfig,\tMethod,\tTotal compressed size\n"); + for (size_t method = 0; methods[method] != NULL; ++method) { + for (size_t datum = 0; data[datum] != NULL; ++datum) { + /* Create the state common to all configs */ + method_state_t* state = methods[method]->create(data[datum]); + for (size_t config = 0; configs[config] != NULL; ++config) { + /* Print the result for the (method, data, config) tuple. */ + result_t const result = + methods[method]->compress(state, configs[config]); + if (result_is_skip(result)) + continue; + tprintf( + results, + "%s,\t%s,\t%s,\t", + data[datum]->name, + configs[config]->name, + methods[method]->name); + if (result_is_error(result)) { + tprintf(results, "%s\n", result_get_error_string(result)); + } else { + tprintf( + results, + "%llu\n", + (unsigned long long)result_get_data(result).total_size); + } + tflush(results); + } + methods[method]->destroy(state); + } + } + return 0; +} + +/** + * Option parsing using getopt. + * When you add a new option update: long_options, long_extras, and + * short_options. + */ + +/** Option variables filled by parse_args. */ +static char const* g_output = NULL; +static char const* g_diff = NULL; +static char const* g_cache = NULL; +static char const* g_zstdcli = NULL; + +typedef enum { + required_option, + optional_option, + help_option, +} option_type; + +/** + * Extra state that we need to keep per-option that we can't store in getopt. + */ +struct option_extra { + int id; /**< The short option name, used as an id. */ + char const* help; /**< The help message. */ + option_type opt_type; /**< The option type: required, optional, or help. */ + char const** value; /**< The value to set or NULL if no_argument. */ +}; + +/** The options. */ +static struct option long_options[] = { + {"cache", required_argument, NULL, 'c'}, + {"diff", required_argument, NULL, 'd'}, + {"help", no_argument, NULL, 'h'}, + {"output", required_argument, NULL, 'o'}, + {"zstd", required_argument, NULL, 'z'}, +}; + +static size_t const nargs = sizeof(long_options) / sizeof(long_options[0]); + +/** The extra info for the options. Must be in the same order as the options. */ +static struct option_extra long_extras[] = { + {'c', "the cache directory", required_option, &g_cache}, + {'d', "compare the results to this file", optional_option, &g_diff}, + {'h', "display this message", help_option, NULL}, + {'o', "write the results here", required_option, &g_output}, + {'z', "zstd cli tool", required_option, &g_zstdcli}, +}; + +/** The short options. Must correspond to the options. */ +static char const short_options[] = "c:d:ho:z:"; + +/** Return the help string for the option type. */ +static char const* required_message(option_type opt_type) { + switch (opt_type) { + case required_option: + return "[required]"; + case optional_option: + return "[optional]"; + case help_option: + return ""; + default: + assert(0); + return NULL; + } +} + +/** Print the help for the program. */ +static void print_help(void) { + fprintf(stderr, "regression test runner\n"); + size_t const nargs = sizeof(long_options) / sizeof(long_options[0]); + for (size_t i = 0; i < nargs; ++i) { + /* Short / long - help [option type] */ + fprintf( + stderr, + "-%c / --%s \t- %s %s\n", + long_options[i].val, + long_options[i].name, + long_extras[i].help, + required_message(long_extras[i].opt_type)); + } +} + +/** Parse the arguments. Teturn 0 on success. Print help on failure. */ +static int parse_args(int argc, char** argv) { + int option_index = 0; + int c; + + while (1) { + c = getopt_long(argc, argv, short_options, long_options, &option_index); + if (c == -1) + break; + + int found = 0; + for (size_t i = 0; i < nargs; ++i) { + if (c == long_extras[i].id && long_extras[i].value != NULL) { + *long_extras[i].value = optarg; + found = 1; + break; + } + } + if (found) + continue; + + switch (c) { + case 'h': + case '?': + default: + print_help(); + return 1; + } + } + + int bad = 0; + for (size_t i = 0; i < nargs; ++i) { + if (long_extras[i].opt_type != required_option) + continue; + if (long_extras[i].value == NULL) + continue; + if (*long_extras[i].value != NULL) + continue; + fprintf( + stderr, + "-%c / --%s is a required argument but is not set\n", + long_options[i].val, + long_options[i].name); + bad = 1; + } + if (bad) { + fprintf(stderr, "\n"); + print_help(); + return 1; + } + + return 0; +} + +/** memcmp() the old results file and the new results file. */ +static int diff_results(char const* actual_file, char const* expected_file) { + data_buffer_t const actual = data_buffer_read(actual_file); + data_buffer_t const expected = data_buffer_read(expected_file); + int ret = 1; + + if (actual.data == NULL) { + fprintf(stderr, "failed to open results '%s' for diff\n", actual_file); + goto out; + } + if (expected.data == NULL) { + fprintf( + stderr, + "failed to open previous results '%s' for diff\n", + expected_file); + goto out; + } + + ret = data_buffer_compare(actual, expected); + if (ret != 0) { + fprintf( + stderr, + "actual results '%s' does not match expected results '%s'\n", + actual_file, + expected_file); + } else { + fprintf(stderr, "actual results match expected results\n"); + } +out: + data_buffer_free(actual); + data_buffer_free(expected); + return ret; +} + +int main(int argc, char** argv) { + /* Parse args and validate modules. */ + int ret = parse_args(argc, argv); + if (ret != 0) + return ret; + + if (are_names_bad()) + return 1; + + /* Initialize modules. */ + method_set_zstdcli(g_zstdcli); + ret = data_init(g_cache); + if (ret != 0) { + fprintf(stderr, "data_init() failed with error=%s\n", strerror(ret)); + return 1; + } + + /* Run the regression tests. */ + ret = 1; + FILE* results = fopen(g_output, "w"); + if (results == NULL) { + fprintf(stderr, "Failed to open the output file\n"); + goto out; + } + ret = run_all(results); + fclose(results); + + if (ret != 0) + goto out; + + if (g_diff) + /* Diff the new results with the previous results. */ + ret = diff_results(g_output, g_diff); + +out: + data_finish(); + return ret; +} From a424899637a437b792bd82dd0c0afb83e3a2b92e Mon Sep 17 00:00:00 2001 From: Mitchell Grenier Date: Fri, 30 Nov 2018 13:20:16 +0000 Subject: [PATCH 11/62] Fix buck for lib --- lib/BUCK | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/BUCK b/lib/BUCK index bd93b082a..637c20d66 100644 --- a/lib/BUCK +++ b/lib/BUCK @@ -1,6 +1,7 @@ cxx_library( name='zstd', header_namespace='', + exported_headers=['zstd.h'], visibility=['PUBLIC'], deps=[ ':common', @@ -17,7 +18,7 @@ cxx_library( exported_headers=subdir_glob([ ('compress', 'zstd*.h'), ]), - srcs=glob(['compress/zstd*.c']), + srcs=glob(['compress/zstd*.c', 'compress/hist.c']), deps=[':common'], ) @@ -40,7 +41,7 @@ cxx_library( header_namespace='', visibility=['PUBLIC'], exported_headers=subdir_glob([ - ('decprecated', '*.h'), + ('deprecated', '*.h'), ]), srcs=glob(['deprecated/*.c']), deps=[':common'], @@ -118,6 +119,7 @@ cxx_library( 'decompress/huf_decompress.c', ], deps=[ + ':debug', ':bitstream', ':compiler', ':errors', @@ -204,9 +206,20 @@ cxx_library( ], ) +cxx_library( + name='debug', + header_namespace='', + visibility=['PUBLIC'], + exported_headers=subdir_glob([ + ('common', 'debug.h'), + ]), + srcs=['common/debug.c'], +) + cxx_library( name='common', deps=[ + ':debug', ':bitstream', ':compiler', ':cpu', From e859862341a7e120ba56773e3c1f5d7bf9911b66 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 30 Nov 2018 17:16:19 -0800 Subject: [PATCH 12/62] [regression] Add dictionary support Dictionaries are prebuilt and saved as part of the data object. The config decides whether or not to use the dictionary if it is available. Configs that require dictionaries are only run with data that have dictionaries. The method will skip configs that are irrelevant, so for example ZSTD_compress() will skip configs with dictionaries. I've also trimmed the silesia source to 1MB per file (12 MB total), and added 500 samples from the github data set with a dictionary. I've intentionally added an extra line to the `results.csv` to make the nightly build fail, so that we can see how CircleCI reports it. Full list of changes: * Add pre-built dictionaries to the data. * Add `use_dictionary` and `no_pledged_src_size` flags to the config. * Add a config using a dictionary for every level. * Add a config that specifies no pledged source size. * Support dictionaries and streaming in the `zstdcli` method. * Add a context-reuse method using `ZSTD_compressCCtx()`. * Clean up the formatting of the `results.csv` file to align columns. * Add `--data`, `--config`, and `--method` flags to constrain each to a particular value. This is useful for debugging a failure or debugging a particular config/method/data. --- tests/regression/config.c | 30 ++++- tests/regression/config.h | 19 +++ tests/regression/data.c | 240 ++++++++++++++++++++++++++--------- tests/regression/data.h | 56 +++++++- tests/regression/method.c | 202 +++++++++++++++++++++-------- tests/regression/results.csv | 144 ++++++++++++++------- tests/regression/test.c | 192 ++++++++++++++++++---------- 7 files changed, 657 insertions(+), 226 deletions(-) diff --git a/tests/regression/config.c b/tests/regression/config.c index 38ac0091b..15713d7eb 100644 --- a/tests/regression/config.c +++ b/tests/regression/config.c @@ -19,6 +19,12 @@ .name = "level -" #x, \ .cli_args = "--fast=" #x, \ .param_values = PARAM_VALUES(level_fast##x##_param_values), \ + }; \ + config_t const level_fast##x##_dict = { \ + .name = "level -" #x " with dict", \ + .cli_args = "--fast=" #x, \ + .param_values = PARAM_VALUES(level_fast##x##_param_values), \ + .use_dictionary = 1, \ }; /* Define a config for each level we want to test with. */ @@ -30,6 +36,12 @@ .name = "level " #x, \ .cli_args = "-" #x, \ .param_values = PARAM_VALUES(level_##x##_param_values), \ + }; \ + config_t const level_##x##_dict = { \ + .name = "level " #x " with dict", \ + .cli_args = "-" #x, \ + .param_values = PARAM_VALUES(level_##x##_param_values), \ + .use_dictionary = 1, \ }; @@ -41,17 +53,31 @@ #undef LEVEL #undef FAST_LEVEL +static config_t no_pledged_src_size = { + .name = "no source size", + .cli_args = "", + .param_values = {.data = NULL, .size = 0}, + .no_pledged_src_size = 1, +}; + static config_t const* g_configs[] = { -#define FAST_LEVEL(x) &level_fast##x, -#define LEVEL(x) &level_##x, + +#define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict, +#define LEVEL(x) &level_##x, &level_##x##_dict, #include "levels.h" #undef LEVEL #undef FAST_LEVEL + + &no_pledged_src_size, NULL, }; config_t const* const* configs = g_configs; +int config_skip_data(config_t const* config, data_t const* data) { + return config->use_dictionary && !data_has_dict(data); +} + int config_get_level(config_t const* config) { param_values_t const params = config->param_values; size_t i; diff --git a/tests/regression/config.h b/tests/regression/config.h index 73f1e9f8a..164f48b55 100644 --- a/tests/regression/config.h +++ b/tests/regression/config.h @@ -16,6 +16,8 @@ #define ZSTD_STATIC_LINKING_ONLY #include +#include "data.h" + typedef struct { ZSTD_cParameter param; unsigned value; @@ -41,8 +43,25 @@ typedef struct { * the parameters will be derived from these. */ param_values_t param_values; + /** + * Boolean parameter that says if we should use a dictionary. If the data + * doesn't have a dictionary, this config is skipped. Defaults to no. + */ + int use_dictionary; + /** + * Boolean parameter that says if we should pass the pledged source size + * when the method allows it. Defaults to yes. + */ + int no_pledged_src_size; } config_t; +/** + * Returns true if the config should skip this data. + * For instance, if the config requires a dictionary but the data doesn't have + * one. + */ +int config_skip_data(config_t const* config, data_t const* data); + #define CONFIG_NO_LEVEL (-ZSTD_TARGETLENGTH_MAX - 1) /** * Returns the compression level specified by the config, or CONFIG_NO_LEVEL if diff --git a/tests/regression/data.c b/tests/regression/data.c index 68269e042..86e7687de 100644 --- a/tests/regression/data.c +++ b/tests/regression/data.c @@ -32,27 +32,58 @@ "https://github.com/facebook/zstd/releases/download/regression-data/" x data_t silesia = { - .url = REGRESSION_RELEASE("silesia.tar.zst"), .name = "silesia", .type = data_type_dir, - .xxhash64 = 0x67558ee5506918b4LL, + .data = + { + .url = REGRESSION_RELEASE("silesia.tar.zst"), + .xxhash64 = 0x48a199f92f93e977LL, + }, }; data_t silesia_tar = { - .url = REGRESSION_RELEASE("silesia.tar.zst"), .name = "silesia.tar", .type = data_type_file, - .xxhash64 = 0x67558ee5506918b4LL, + .data = + { + .url = REGRESSION_RELEASE("silesia.tar.zst"), + .xxhash64 = 0x48a199f92f93e977LL, + }, +}; + +data_t github = { + .name = "github", + .type = data_type_dir, + .data = + { + .url = REGRESSION_RELEASE("github.tar.zst"), + .xxhash64 = 0xa9b1b44b020df292LL, + }, + .dict = + { + .url = REGRESSION_RELEASE("github.dict.zst"), + .xxhash64 = 0x1eddc6f737d3cb53LL, + + }, }; static data_t* g_data[] = { &silesia, &silesia_tar, + &github, NULL, }; data_t const* const* data = (data_t const* const*)g_data; +/** + * data helpers. + */ + +int data_has_dict(data_t const* data) { + return data->dict.url != NULL; +} + /** * data buffer helper functions (documented in header). */ @@ -100,16 +131,24 @@ err: free(buffer.data); memset(&buffer, 0, sizeof(buffer)); return buffer; - } -data_buffer_t data_buffer_get(data_t const* data) { +data_buffer_t data_buffer_get_data(data_t const* data) { data_buffer_t const kEmptyBuffer = {}; if (data->type != data_type_file) return kEmptyBuffer; - return data_buffer_read(data->path); + return data_buffer_read(data->data.path); +} + +data_buffer_t data_buffer_get_dict(data_t const* data) { + data_buffer_t const kEmptyBuffer = {}; + + if (!data_has_dict(data)) + return kEmptyBuffer; + + return data_buffer_read(data->dict.path); } int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2) { @@ -124,13 +163,69 @@ int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2) { return 0; assert(buffer1.size > buffer2.size); return 1; - } void data_buffer_free(data_buffer_t buffer) { free(buffer.data); } +/** + * data filenames helpers. + */ + +data_filenames_t data_filenames_get(data_t const* data) { + data_filenames_t filenames = {.buffer = NULL, .size = 0}; + char const* path = data->data.path; + + filenames.filenames = UTIL_createFileList( + &path, + 1, + &filenames.buffer, + &filenames.size, + /* followLinks */ 0); + return filenames; +} + +void data_filenames_free(data_filenames_t filenames) { + UTIL_freeFileList(filenames.filenames, filenames.buffer); +} + +/** + * data buffers helpers. + */ + +data_buffers_t data_buffers_get(data_t const* data) { + data_buffers_t buffers = {.size = 0}; + data_filenames_t filenames = data_filenames_get(data); + if (filenames.size == 0) + return buffers; + + data_buffer_t* buffersPtr = + (data_buffer_t*)malloc(filenames.size * sizeof(data_buffer_t)); + if (buffersPtr == NULL) + return buffers; + buffers.buffers = (data_buffer_t const*)buffersPtr; + buffers.size = filenames.size; + + for (size_t i = 0; i < filenames.size; ++i) { + buffersPtr[i] = data_buffer_read(filenames.filenames[i]); + if (buffersPtr[i].data == NULL) { + data_buffers_t const kEmptyBuffer = {}; + data_buffers_free(buffers); + return kEmptyBuffer; + } + } + + return buffers; +} + +/** + * Frees the data buffers. + */ +void data_buffers_free(data_buffers_t buffers) { + free((data_buffer_t*)buffers.buffers); +} + /** * Initialization and download functions. */ @@ -174,18 +269,23 @@ out: static char* cat3(char const* str1, char const* str2, char const* str3) { size_t const size1 = strlen(str1); size_t const size2 = strlen(str2); - size_t const size3 = strlen(str3); + size_t const size3 = str3 == NULL ? 0 : strlen(str3); size_t const size = size1 + size2 + size3 + 1; char* const dst = (char*)malloc(size); if (dst == NULL) return NULL; strcpy(dst, str1); strcpy(dst + size1, str2); - strcpy(dst + size1 + size2, str3); + if (str3 != NULL) + strcpy(dst + size1 + size2, str3); assert(strlen(dst) == size1 + size2 + size3); return dst; } +static char* cat2(char const* str1, char const* str2) { + return cat3(str1, str2, NULL); +} + /** * State needed by the curl callback. * It takes data from curl, hashes it, and writes it to the file. @@ -197,16 +297,18 @@ typedef struct { } curl_data_t; /** Create the curl state. */ -static curl_data_t curl_data_create(data_t const* data) { +static curl_data_t curl_data_create( + data_resource_t const* resource, + data_type_t type) { curl_data_t cdata = {}; XXH64_reset(&cdata.xxhash64, 0); assert(UTIL_isDirectory(g_data_dir)); - if (data->type == data_type_file) { + if (type == data_type_file) { /* Decompress the resource and store to the path. */ - char* cmd = cat3("zstd -dqfo '", data->path, "'"); + char* cmd = cat3("zstd -dqfo '", resource->path, "'"); if (cmd == NULL) { cdata.error = ENOMEM; return cdata; @@ -243,54 +345,68 @@ static size_t curl_write(void* data, size_t size, size_t count, void* ptr) { return written; } -/** Download a single data object. */ -static int curl_download_datum(CURL* curl, data_t const* data) { - curl_data_t cdata = curl_data_create(data); - int err = EFAULT; - - if (cdata.error != 0) { - err = cdata.error; - goto out; - } - +static int curl_download_resource( + CURL* curl, + data_resource_t const* resource, + data_type_t type) { + curl_data_t cdata; /* Download the data. */ - if (curl_easy_setopt(curl, CURLOPT_URL, data->url) != 0) - goto out; + if (curl_easy_setopt(curl, CURLOPT_URL, resource->url) != 0) + return EINVAL; if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cdata) != 0) - goto out; - if (curl_easy_perform(curl) != 0) { - fprintf(stderr, "downloading '%s' failed\n", data->url); - goto out; - } - /* check that the file exists. */ - if (data->type == data_type_file && !UTIL_isRegularFile(data->path)) { - fprintf(stderr, "output file '%s' does not exist\n", data->path); - goto out; - } - if (data->type == data_type_dir && !UTIL_isDirectory(data->path)) { - fprintf(stderr, "output directory '%s' does not exist\n", data->path); - goto out; - } - /* Check that the hash matches. */ - if (XXH64_digest(&cdata.xxhash64) != data->xxhash64) { + return EINVAL; + cdata = curl_data_create(resource, type); + if (cdata.error != 0) + return cdata.error; + int const curl_err = curl_easy_perform(curl); + int const close_err = curl_data_free(cdata); + if (curl_err) { fprintf( stderr, - "checksum does not match: %llx != %llx\n", + "downloading '%s' for '%s' failed\n", + resource->url, + resource->path); + return EIO; + } + if (close_err) { + fprintf(stderr, "writing data to '%s' failed\n", resource->path); + return EIO; + } + /* check that the file exists. */ + if (type == data_type_file && !UTIL_isRegularFile(resource->path)) { + fprintf(stderr, "output file '%s' does not exist\n", resource->path); + return EIO; + } + if (type == data_type_dir && !UTIL_isDirectory(resource->path)) { + fprintf( + stderr, "output directory '%s' does not exist\n", resource->path); + return EIO; + } + /* Check that the hash matches. */ + if (XXH64_digest(&cdata.xxhash64) != resource->xxhash64) { + fprintf( + stderr, + "checksum does not match: 0x%llxLL != 0x%llxLL\n", (unsigned long long)XXH64_digest(&cdata.xxhash64), - (unsigned long long)data->xxhash64); - goto out; + (unsigned long long)resource->xxhash64); + return EINVAL; } - err = 0; -out: - if (err != 0) - fprintf(stderr, "downloading '%s' failed\n", data->name); - int const close_err = curl_data_free(cdata); - if (close_err != 0 && err == 0) { - fprintf(stderr, "failed to write data for '%s'\n", data->name); - err = close_err; + return 0; +} + +/** Download a single data object. */ +static int curl_download_datum(CURL* curl, data_t const* data) { + int ret; + ret = curl_download_resource(curl, &data->data, data->type); + if (ret != 0) + return ret; + if (data_has_dict(data)) { + ret = curl_download_resource(curl, &data->dict, data_type_file); + if (ret != 0) + return ret; } - return err; + return ret; } /** Download all the data. */ @@ -331,9 +447,14 @@ static int data_create_paths(data_t* const* data, char const* dir) { assert(data != NULL); for (; *data != NULL; ++data) { data_t* const datum = *data; - datum->path = cat3(dir, "/", datum->name); - if (datum->path == NULL) + datum->data.path = cat3(dir, "/", datum->name); + if (datum->data.path == NULL) return ENOMEM; + if (data_has_dict(datum)) { + datum->dict.path = cat2(datum->data.path, ".dict"); + if (datum->dict.path == NULL) + return ENOMEM; + } } return 0; } @@ -343,8 +464,10 @@ static void data_free_paths(data_t* const* data) { assert(data != NULL); for (; *data != NULL; ++data) { data_t* datum = *data; - free((void*)datum->path); - datum->path = NULL; + free((void*)datum->data.path); + free((void*)datum->dict.path); + datum->data.path = NULL; + datum->dict.path = NULL; } } @@ -367,7 +490,8 @@ static uint64_t stamp_hash(data_t const* const* data) { /* We don't care about the URL that we fetch from. */ /* The path is derived from the name. */ XXH64_update(&state, datum->name, strlen(datum->name)); - xxh_update_le(&state, datum->xxhash64); + xxh_update_le(&state, datum->data.xxhash64); + xxh_update_le(&state, datum->dict.xxhash64); xxh_update_le(&state, datum->type); } return XXH64_digest(&state); diff --git a/tests/regression/data.h b/tests/regression/data.h index d75f645b1..717fe1294 100644 --- a/tests/regression/data.h +++ b/tests/regression/data.h @@ -22,10 +22,14 @@ typedef enum { typedef struct { char const* url; /**< Where to get this resource. */ uint64_t xxhash64; /**< Hash of the url contents. */ - char const* name; /**< The logical name of the resource (no extension). */ - data_type_t type; /**< The type of this resource. */ char const* path; /**< The path of the unpacked resource (derived). */ - size_t size; +} data_resource_t; + +typedef struct { + data_resource_t data; + data_resource_t dict; + data_type_t type; /**< The type of the data. */ + char const* name; /**< The logical name of the data (no extension). */ } data_t; /** @@ -33,6 +37,9 @@ typedef struct { */ extern data_t const* const* data; + +int data_has_dict(data_t const* data); + /** * Initializes the data module and downloads the data necessary. * Caches the downloads in dir. We add a stamp file in the directory after @@ -62,7 +69,14 @@ typedef struct { * * @returns The buffer, which is NULL on failure. */ -data_buffer_t data_buffer_get(data_t const* data); +data_buffer_t data_buffer_get_data(data_t const* data); + +/** + * Read the dictionary that the data points to into a buffer. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_get_dict(data_t const* data); /** * Read the contents of filename into a buffer. @@ -88,5 +102,39 @@ int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2); */ void data_buffer_free(data_buffer_t buffer); +typedef struct { + char* buffer; + char const** filenames; + unsigned size; +} data_filenames_t; + +/** + * Get a recursive list of filenames in the data object. If it is a file, it + * will only contain one entry. If it is a directory, it will recursively walk + * the directory. + * + * @returns The list of filenames, which has size 0 and NULL pointers on error. + */ +data_filenames_t data_filenames_get(data_t const* data); + +/** + * Frees the filenames table. + */ +void data_filenames_free(data_filenames_t filenames); + +typedef struct { + data_buffer_t const* buffers; + size_t size; +} data_buffers_t; + +/** + * @returns a list of buffers for every file in data. It is zero sized on error. + */ +data_buffers_t data_buffers_get(data_t const* data); + +/** + * Frees the data buffers. + */ +void data_buffers_free(data_buffers_t buffers); #endif diff --git a/tests/regression/method.c b/tests/regression/method.c index c19a377a7..3fedcbe78 100644 --- a/tests/regression/method.c +++ b/tests/regression/method.c @@ -26,62 +26,88 @@ void method_set_zstdcli(char const* zstdcli) { * the given name, member. * * method_state_t* base = ...; - * simple_state_t* state = container_of(base, simple_state_t, base); + * buffer_state_t* state = container_of(base, buffer_state_t, base); */ #define container_of(ptr, type, member) \ - ((type*)(char*)(ptr)-offsetof(type, member)) + ((type*)(ptr == NULL ? NULL : (char*)(ptr)-offsetof(type, member))) /** State to reuse the same buffers between compression calls. */ typedef struct { method_state_t base; - data_buffer_t buffer; /**< The constant input data buffer. */ - data_buffer_t compressed; /**< The compressed data buffer. */ - data_buffer_t decompressed; /**< The decompressed data buffer. */ -} simple_state_t; + data_buffers_t inputs; /**< The input buffer for each file. */ + data_buffer_t compressed; /**< The compressed data buffer. */ + data_buffer_t decompressed; /**< The decompressed data buffer. */ +} buffer_state_t; -static method_state_t* simple_create(data_t const* data) { - simple_state_t* state = (simple_state_t*)calloc(1, sizeof(simple_state_t)); +static size_t buffers_max_size(data_buffers_t buffers) { + size_t max = 0; + for (size_t i = 0; i < buffers.size; ++i) { + if (buffers.buffers[i].size > max) + max = buffers.buffers[i].size; + } + return max; +} + +static method_state_t* buffer_state_create(data_t const* data) { + buffer_state_t* state = (buffer_state_t*)calloc(1, sizeof(buffer_state_t)); if (state == NULL) return NULL; state->base.data = data; - state->buffer = data_buffer_get(data); - state->compressed = - data_buffer_create(ZSTD_compressBound(state->buffer.size)); - state->decompressed = data_buffer_create(state->buffer.size); + state->inputs = data_buffers_get(data); + size_t const max_size = buffers_max_size(state->inputs); + state->compressed = data_buffer_create(ZSTD_compressBound(max_size)); + state->decompressed = data_buffer_create(max_size); return &state->base; } -static void simple_destroy(method_state_t* base) { +static void buffer_state_destroy(method_state_t* base) { if (base == NULL) return; - simple_state_t* state = container_of(base, simple_state_t, base); + buffer_state_t* state = container_of(base, buffer_state_t, base); free(state); } -static result_t simple_compress(method_state_t* base, config_t const* config) { - if (base == NULL) - return result_error(result_error_system_error); - simple_state_t* state = container_of(base, simple_state_t, base); +static int buffer_state_bad(buffer_state_t const* state) { + if (state == NULL) { + fprintf(stderr, "buffer_state_t is NULL\n"); + return 1; + } + if (state->inputs.size == 0 || state->compressed.data == NULL || + state->decompressed.data == NULL) { + fprintf(stderr, "buffer state allocation failure\n"); + return 1; + } + return 0; +} +static result_t simple_compress(method_state_t* base, config_t const* config) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state)) + return result_error(result_error_system_error); + + /* Keep the tests short by skipping directories, since behavior shouldn't + * change. + */ if (base->data->type != data_type_file) return result_error(result_error_skip); - if (state->buffer.data == NULL || state->compressed.data == NULL || - state->decompressed.data == NULL) { - return result_error(result_error_system_error); - } + if (config->use_dictionary || config->no_pledged_src_size) + return result_error(result_error_skip); /* If the config doesn't specify a level, skip. */ int const level = config_get_level(config); if (level == CONFIG_NO_LEVEL) return result_error(result_error_skip); + data_buffer_t const input = state->inputs.buffers[0]; + /* Compress, decompress, and check the result. */ state->compressed.size = ZSTD_compress( state->compressed.data, state->compressed.capacity, - state->buffer.data, - state->buffer.size, + input.data, + input.size, level); if (ZSTD_isError(state->compressed.size)) return result_error(result_error_compression_error); @@ -93,7 +119,7 @@ static result_t simple_compress(method_state_t* base, config_t const* config) { state->compressed.size); if (ZSTD_isError(state->decompressed.size)) return result_error(result_error_decompression_error); - if (data_buffer_compare(state->buffer, state->decompressed)) + if (data_buffer_compare(input, state->decompressed)) return result_error(result_error_round_trip_error); result_data_t data; @@ -101,6 +127,70 @@ static result_t simple_compress(method_state_t* base, config_t const* config) { return result_data(data); } +static result_t compress_cctx_compress( + method_state_t* base, + config_t const* config) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state)) + return result_error(result_error_system_error); + + if (config->use_dictionary || config->no_pledged_src_size) + return result_error(result_error_skip); + + if (base->data->type != data_type_dir) + return result_error(result_error_skip); + + int const level = config_get_level(config); + if (level == CONFIG_NO_LEVEL) + return result_error(result_error_skip); + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + if (cctx == NULL) { + fprintf(stderr, "ZSTD_createCCtx() failed\n"); + return result_error(result_error_system_error); + } + + result_t result; + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t const input = state->inputs.buffers[i]; + + state->compressed.size = ZSTD_compressCCtx( + cctx, + state->compressed.data, + state->compressed.capacity, + input.data, + input.size, + level); + if (ZSTD_isError(state->compressed.size)) { + result = result_error(result_error_compression_error); + goto out; + } + + state->decompressed.size = ZSTD_decompress( + state->decompressed.data, + state->decompressed.capacity, + state->compressed.data, + state->compressed.size); + if (ZSTD_isError(state->decompressed.size)) { + result = result_error(result_error_decompression_error); + goto out; + } + if (data_buffer_compare(input, state->decompressed)) { + result = result_error(result_error_round_trip_error); + goto out; + } + + data.total_size += state->compressed.size; + } + + result = result_data(data); +out: + ZSTD_freeCCtx(cctx); + return result; +} + /** Generic state creation function. */ static method_state_t* method_state_create(data_t const* data) { method_state_t* state = (method_state_t*)malloc(sizeof(method_state_t)); @@ -114,26 +204,32 @@ static void method_state_destroy(method_state_t* state) { free(state); } -#define MAX_OUT 32 - -static result_t cli_file_compress( +static result_t cli_compress( method_state_t* state, config_t const* config) { if (config->cli_args == NULL) return result_error(result_error_skip); + /* We don't support no pledged source size with directories. Too slow. */ + if (state->data->type == data_type_dir && config->no_pledged_src_size) + return result_error(result_error_skip); + if (g_zstdcli == NULL) return result_error(result_error_system_error); - /* '' -r '' | wc -c */ + /* '' -cqr [-D ''] '' */ char cmd[1024]; size_t const cmd_size = snprintf( cmd, sizeof(cmd), - "'%s' -cqr %s '%s' | wc -c", + "'%s' -cqr %s %s%s%s %s '%s'", g_zstdcli, config->cli_args, - state->data->path); + config->use_dictionary ? "-D '" : "", + config->use_dictionary ? state->data->dict.path : "", + config->use_dictionary ? "'" : "", + config->no_pledged_src_size ? "<" : "", + state->data->data.path); if (cmd_size >= sizeof(cmd)) { fprintf(stderr, "command too large: %s\n", cmd); return result_error(result_error_system_error); @@ -144,42 +240,48 @@ static result_t cli_file_compress( return result_error(result_error_system_error); } - /* Read the total compressed size. */ - char out[MAX_OUT + 1]; - size_t const out_size = fread(out, 1, MAX_OUT, zstd); - out[out_size] = '\0'; - int const zstd_ret = pclose(zstd); - if (zstd_ret != 0) { + char out[4096]; + size_t total_size = 0; + while (1) { + size_t const size = fread(out, 1, sizeof(out), zstd); + total_size += size; + if (size != sizeof(out)) + break; + } + if (ferror(zstd) || pclose(zstd) != 0) { fprintf(stderr, "zstd failed with command: %s\n", cmd); return result_error(result_error_compression_error); } - if (out_size == MAX_OUT) { - fprintf(stderr, "wc -c produced more bytes than expected: %s\n", out); - return result_error(result_error_system_error); - } - result_data_t data; - data.total_size = atoll(out); + result_data_t const data = {.total_size = total_size}; return result_data(data); } method_t const simple = { - .name = "simple", - .create = simple_create, + .name = "ZSTD_compress", + .create = buffer_state_create, .compress = simple_compress, - .destroy = simple_destroy, + .destroy = buffer_state_destroy, }; -method_t const cli_file = { - .name = "cli file", +method_t const compress_cctx = { + .name = "ZSTD_compressCCtx", + .create = buffer_state_create, + .compress = compress_cctx_compress, + .destroy = buffer_state_destroy, +}; + +method_t const cli = { + .name = "zstdcli", .create = method_state_create, - .compress = cli_file_compress, + .compress = cli_compress, .destroy = method_state_destroy, }; static method_t const* g_methods[] = { &simple, - &cli_file, + &compress_cctx, + &cli, NULL, }; diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 8ed689d26..d9f1873ce 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -1,43 +1,101 @@ -Data, Config, Method, Total compressed size -silesia.tar, level -5, simple, 106176430 -silesia.tar, level -3, simple, 98476550 -silesia.tar, level -1, simple, 87206767 -silesia.tar, level 0, simple, 66996953 -silesia.tar, level 1, simple, 73658303 -silesia.tar, level 3, simple, 66996953 -silesia.tar, level 4, simple, 65996020 -silesia.tar, level 5, simple, 64421326 -silesia.tar, level 6, simple, 62388673 -silesia.tar, level 7, simple, 61159525 -silesia.tar, level 9, simple, 60214921 -silesia.tar, level 13, simple, 58428642 -silesia.tar, level 16, simple, 56363759 -silesia.tar, level 19, simple, 53274173 -silesia, level -5, cli file, 106202112 -silesia, level -3, cli file, 98518660 -silesia, level -1, cli file, 87226203 -silesia, level 0, cli file, 67049190 -silesia, level 1, cli file, 73676282 -silesia, level 3, cli file, 67049190 -silesia, level 4, cli file, 66090040 -silesia, level 5, cli file, 64503721 -silesia, level 6, cli file, 62446177 -silesia, level 7, cli file, 61217029 -silesia, level 9, cli file, 60282841 -silesia, level 13, cli file, 58480658 -silesia, level 16, cli file, 56414170 -silesia, level 19, cli file, 53365292 -silesia.tar, level -5, cli file, 106250113 -silesia.tar, level -3, cli file, 98550747 -silesia.tar, level -1, cli file, 87227322 -silesia.tar, level 0, cli file, 67111168 -silesia.tar, level 1, cli file, 73694374 -silesia.tar, level 3, cli file, 67111168 -silesia.tar, level 4, cli file, 66154079 -silesia.tar, level 5, cli file, 64546998 -silesia.tar, level 6, cli file, 62458454 -silesia.tar, level 7, cli file, 61231085 -silesia.tar, level 9, cli file, 60310313 -silesia.tar, level 13, cli file, 58517476 -silesia.tar, level 16, cli file, 56448694 -silesia.tar, level 19, cli file, 53444920 +Data, Config, Method, Total compressed size +This line is intentionally added to see how the nightly job reports failures +silesia.tar, level -5, ZSTD_compress, 7160438 +silesia.tar, level -3, ZSTD_compress, 6789024 +silesia.tar, level -1, ZSTD_compress, 6195462 +silesia.tar, level 0, ZSTD_compress, 4875071 +silesia.tar, level 1, ZSTD_compress, 5339697 +silesia.tar, level 3, ZSTD_compress, 4875071 +silesia.tar, level 4, ZSTD_compress, 4813104 +silesia.tar, level 5, ZSTD_compress, 4726961 +silesia.tar, level 6, ZSTD_compress, 4654401 +silesia.tar, level 7, ZSTD_compress, 4591933 +silesia.tar, level 9, ZSTD_compress, 4554098 +silesia.tar, level 13, ZSTD_compress, 4503496 +silesia.tar, level 16, ZSTD_compress, 4387233 +silesia.tar, level 19, ZSTD_compress, 4283123 +silesia, level -5, ZSTD_compressCCtx, 7152294 +silesia, level -3, ZSTD_compressCCtx, 6789969 +silesia, level -1, ZSTD_compressCCtx, 6191548 +silesia, level 0, ZSTD_compressCCtx, 4862377 +silesia, level 1, ZSTD_compressCCtx, 5318036 +silesia, level 3, ZSTD_compressCCtx, 4862377 +silesia, level 4, ZSTD_compressCCtx, 4800629 +silesia, level 5, ZSTD_compressCCtx, 4715005 +silesia, level 6, ZSTD_compressCCtx, 4644055 +silesia, level 7, ZSTD_compressCCtx, 4581559 +silesia, level 9, ZSTD_compressCCtx, 4543862 +silesia, level 13, ZSTD_compressCCtx, 4493931 +silesia, level 16, ZSTD_compressCCtx, 4381885 +silesia, level 19, ZSTD_compressCCtx, 4296899 +github, level -5, ZSTD_compressCCtx, 232744 +github, level -3, ZSTD_compressCCtx, 220611 +github, level -1, ZSTD_compressCCtx, 176575 +github, level 0, ZSTD_compressCCtx, 136397 +github, level 1, ZSTD_compressCCtx, 143457 +github, level 3, ZSTD_compressCCtx, 136397 +github, level 4, ZSTD_compressCCtx, 136144 +github, level 5, ZSTD_compressCCtx, 135106 +github, level 6, ZSTD_compressCCtx, 135108 +github, level 7, ZSTD_compressCCtx, 135108 +github, level 9, ZSTD_compressCCtx, 135108 +github, level 13, ZSTD_compressCCtx, 133741 +github, level 16, ZSTD_compressCCtx, 133741 +github, level 19, ZSTD_compressCCtx, 133717 +silesia, level -5, zstdcli, 7152342 +silesia, level -3, zstdcli, 6790021 +silesia, level -1, zstdcli, 6191597 +silesia, level 0, zstdcli, 4862425 +silesia, level 1, zstdcli, 5318084 +silesia, level 3, zstdcli, 4862425 +silesia, level 4, zstdcli, 4800677 +silesia, level 5, zstdcli, 4715053 +silesia, level 6, zstdcli, 4644103 +silesia, level 7, zstdcli, 4581607 +silesia, level 9, zstdcli, 4543910 +silesia, level 13, zstdcli, 4493979 +silesia, level 16, zstdcli, 4381933 +silesia, level 19, zstdcli, 4296947 +silesia.tar, level -5, zstdcli, 7159586 +silesia.tar, level -3, zstdcli, 6791018 +silesia.tar, level -1, zstdcli, 6196283 +silesia.tar, level 0, zstdcli, 4876730 +silesia.tar, level 1, zstdcli, 5340312 +silesia.tar, level 3, zstdcli, 4876730 +silesia.tar, level 4, zstdcli, 4817723 +silesia.tar, level 5, zstdcli, 4730389 +silesia.tar, level 6, zstdcli, 4655708 +silesia.tar, level 7, zstdcli, 4593407 +silesia.tar, level 9, zstdcli, 4556135 +silesia.tar, level 13, zstdcli, 4503500 +silesia.tar, level 16, zstdcli, 4387237 +silesia.tar, level 19, zstdcli, 4283127 +silesia.tar, no source size, zstdcli, 4876726 +github, level -5, zstdcli, 234744 +github, level -5 with dict, zstdcli, 47528 +github, level -3, zstdcli, 222611 +github, level -3 with dict, zstdcli, 46394 +github, level -1, zstdcli, 178575 +github, level -1 with dict, zstdcli, 43401 +github, level 0, zstdcli, 138397 +github, level 0 with dict, zstdcli, 40316 +github, level 1, zstdcli, 145457 +github, level 1 with dict, zstdcli, 43242 +github, level 3, zstdcli, 138397 +github, level 3 with dict, zstdcli, 40316 +github, level 4, zstdcli, 138144 +github, level 4 with dict, zstdcli, 40292 +github, level 5, zstdcli, 137106 +github, level 5 with dict, zstdcli, 40938 +github, level 6, zstdcli, 137108 +github, level 6 with dict, zstdcli, 40632 +github, level 7, zstdcli, 137108 +github, level 7 with dict, zstdcli, 40766 +github, level 9, zstdcli, 137108 +github, level 9 with dict, zstdcli, 41326 +github, level 13, zstdcli, 135741 +github, level 13 with dict, zstdcli, 41670 +github, level 16, zstdcli, 135741 +github, level 16 with dict, zstdcli, 39940 +github, level 19, zstdcli, 135717 +github, level 19 with dict, zstdcli, 39576 diff --git a/tests/regression/test.c b/tests/regression/test.c index c3063511c..9e7b83c19 100644 --- a/tests/regression/test.c +++ b/tests/regression/test.c @@ -17,10 +17,15 @@ #include "data.h" #include "method.h" -/** Check if a name contains a comma. */ +static int g_max_name_len = 0; + +/** Check if a name contains a comma or is too long. */ static int is_name_bad(char const* name) { if (name == NULL) return 1; + int const len = strlen(name); + if (len > g_max_name_len) + g_max_name_len = len; for (; *name != '\0'; ++name) if (*name == ',') return 1; @@ -47,57 +52,6 @@ static int are_names_bad() { return 0; } -/** Helper macro to print to stderr and a file. */ -#define tprintf(file, ...) \ - do { \ - fprintf(file, __VA_ARGS__); \ - fprintf(stderr, __VA_ARGS__); \ - } while (0) -/** Helper macro to flush stderr and a file. */ -#define tflush(file) \ - do { \ - fflush(file); \ - fflush(stderr); \ - } while (0) - -/** - * Run all the regression tests and record the results table to results and - * stderr progressively. - */ -static int run_all(FILE* results) { - tprintf(results, "Data,\tConfig,\tMethod,\tTotal compressed size\n"); - for (size_t method = 0; methods[method] != NULL; ++method) { - for (size_t datum = 0; data[datum] != NULL; ++datum) { - /* Create the state common to all configs */ - method_state_t* state = methods[method]->create(data[datum]); - for (size_t config = 0; configs[config] != NULL; ++config) { - /* Print the result for the (method, data, config) tuple. */ - result_t const result = - methods[method]->compress(state, configs[config]); - if (result_is_skip(result)) - continue; - tprintf( - results, - "%s,\t%s,\t%s,\t", - data[datum]->name, - configs[config]->name, - methods[method]->name); - if (result_is_error(result)) { - tprintf(results, "%s\n", result_get_error_string(result)); - } else { - tprintf( - results, - "%llu\n", - (unsigned long long)result_get_data(result).total_size); - } - tflush(results); - } - methods[method]->destroy(state); - } - } - return 0; -} - /** * Option parsing using getopt. * When you add a new option update: long_options, long_extras, and @@ -109,6 +63,9 @@ static char const* g_output = NULL; static char const* g_diff = NULL; static char const* g_cache = NULL; static char const* g_zstdcli = NULL; +static char const* g_config = NULL; +static char const* g_data = NULL; +static char const* g_method = NULL; typedef enum { required_option, @@ -120,19 +77,22 @@ typedef enum { * Extra state that we need to keep per-option that we can't store in getopt. */ struct option_extra { - int id; /**< The short option name, used as an id. */ - char const* help; /**< The help message. */ + int id; /**< The short option name, used as an id. */ + char const* help; /**< The help message. */ option_type opt_type; /**< The option type: required, optional, or help. */ - char const** value; /**< The value to set or NULL if no_argument. */ + char const** value; /**< The value to set or NULL if no_argument. */ }; /** The options. */ static struct option long_options[] = { {"cache", required_argument, NULL, 'c'}, - {"diff", required_argument, NULL, 'd'}, - {"help", no_argument, NULL, 'h'}, {"output", required_argument, NULL, 'o'}, {"zstd", required_argument, NULL, 'z'}, + {"config", required_argument, NULL, 128}, + {"data", required_argument, NULL, 129}, + {"method", required_argument, NULL, 130}, + {"diff", required_argument, NULL, 'd'}, + {"help", no_argument, NULL, 'h'}, }; static size_t const nargs = sizeof(long_options) / sizeof(long_options[0]); @@ -140,10 +100,13 @@ static size_t const nargs = sizeof(long_options) / sizeof(long_options[0]); /** The extra info for the options. Must be in the same order as the options. */ static struct option_extra long_extras[] = { {'c', "the cache directory", required_option, &g_cache}, - {'d', "compare the results to this file", optional_option, &g_diff}, - {'h', "display this message", help_option, NULL}, {'o', "write the results here", required_option, &g_output}, {'z', "zstd cli tool", required_option, &g_zstdcli}, + {128, "use this config", optional_option, &g_config}, + {129, "use this data", optional_option, &g_data}, + {130, "use this method", optional_option, &g_method}, + {'d', "compare the results to this file", optional_option, &g_diff}, + {'h', "display this message", help_option, NULL}, }; /** The short options. Must correspond to the options. */ @@ -169,14 +132,24 @@ static void print_help(void) { fprintf(stderr, "regression test runner\n"); size_t const nargs = sizeof(long_options) / sizeof(long_options[0]); for (size_t i = 0; i < nargs; ++i) { - /* Short / long - help [option type] */ - fprintf( - stderr, - "-%c / --%s \t- %s %s\n", - long_options[i].val, - long_options[i].name, - long_extras[i].help, - required_message(long_extras[i].opt_type)); + if (long_options[i].val < 128) { + /* Long / short - help [option type] */ + fprintf( + stderr, + "--%s / -%c \t- %s %s\n", + long_options[i].name, + long_options[i].val, + long_extras[i].help, + required_message(long_extras[i].opt_type)); + } else { + /* Short / long - help [option type] */ + fprintf( + stderr, + "--%s \t- %s %s\n", + long_options[i].name, + long_extras[i].help, + required_message(long_extras[i].opt_type)); + } } } @@ -220,8 +193,7 @@ static int parse_args(int argc, char** argv) { continue; fprintf( stderr, - "-%c / --%s is a required argument but is not set\n", - long_options[i].val, + "--%s is a required argument but is not set\n", long_options[i].name); bad = 1; } @@ -234,6 +206,88 @@ static int parse_args(int argc, char** argv) { return 0; } +/** Helper macro to print to stderr and a file. */ +#define tprintf(file, ...) \ + do { \ + fprintf(file, __VA_ARGS__); \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) +/** Helper macro to flush stderr and a file. */ +#define tflush(file) \ + do { \ + fflush(file); \ + fflush(stderr); \ + } while (0) + +void tprint_names( + FILE* results, + char const* data_name, + char const* config_name, + char const* method_name) { + int const data_padding = g_max_name_len - strlen(data_name); + int const config_padding = g_max_name_len - strlen(config_name); + int const method_padding = g_max_name_len - strlen(method_name); + + tprintf( + results, + "%s, %*s%s, %*s%s, %*s", + data_name, + data_padding, + "", + config_name, + config_padding, + "", + method_name, + method_padding, + ""); +} + +/** + * Run all the regression tests and record the results table to results and + * stderr progressively. + */ +static int run_all(FILE* results) { + tprint_names(results, "Data", "Config", "Method"); + tprintf(results, "Total compressed size\n"); + for (size_t method = 0; methods[method] != NULL; ++method) { + if (g_method != NULL && strcmp(methods[method]->name, g_method)) + continue; + for (size_t datum = 0; data[datum] != NULL; ++datum) { + if (g_data != NULL && strcmp(data[datum]->name, g_data)) + continue; + /* Create the state common to all configs */ + method_state_t* state = methods[method]->create(data[datum]); + for (size_t config = 0; configs[config] != NULL; ++config) { + if (g_config != NULL && strcmp(configs[config]->name, g_config)) + continue; + if (config_skip_data(configs[config], data[datum])) + continue; + /* Print the result for the (method, data, config) tuple. */ + result_t const result = + methods[method]->compress(state, configs[config]); + if (result_is_skip(result)) + continue; + tprint_names( + results, + data[datum]->name, + configs[config]->name, + methods[method]->name); + if (result_is_error(result)) { + tprintf(results, "%s\n", result_get_error_string(result)); + } else { + tprintf( + results, + "%llu\n", + (unsigned long long)result_get_data(result).total_size); + } + tflush(results); + } + methods[method]->destroy(state); + } + } + return 0; +} + /** memcmp() the old results file and the new results file. */ static int diff_results(char const* actual_file, char const* expected_file) { data_buffer_t const actual = data_buffer_read(actual_file); From f7278087319917339826613cb7d0e1e9ba263f09 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 02:47:04 +0700 Subject: [PATCH 13/62] Minor fix for meson build Use files function instead of constructing path with meson.current_source_dir() --- contrib/meson/contrib/gen_html/meson.build | 2 +- contrib/meson/meson.build | 9 ++++++++- contrib/meson/meson_options.txt | 2 +- contrib/meson/programs/meson.build | 3 --- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/contrib/meson/contrib/gen_html/meson.build b/contrib/meson/contrib/gen_html/meson.build index 086b8f058..38ed52761 100644 --- a/contrib/meson/contrib/gen_html/meson.build +++ b/contrib/meson/contrib/gen_html/meson.build @@ -16,7 +16,7 @@ contrib_gen_html_dir = join_paths(zstd_source_dir, 'contrib', 'gen_html') gen_html_includes = include_directories(programs_dir, library_dir, library_common_dir, - contrib_gen_html_dir ) + contrib_gen_html_dir) gen_html = executable('gen_html', join_paths(contrib_gen_html_dir, 'gen_html.cpp'), diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index b9ac88281..48f8d570d 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -52,10 +52,17 @@ enable_zlib = get_option('zlib_support') enable_lzma = get_option('lzma_support') # ============================================================================= -# Getting project version from zstd.h +# Helper scripts for Meson # ============================================================================= GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py') +CreateSymlink_py = files('CreateSymlink.py') +CopyFile_py = files('CopyFile.py') + +# ============================================================================= +# Getting project version from zstd.h +# ============================================================================= + zstd_h_file = join_paths(library_dir, 'zstd.h') r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file) if r.returncode() == 0 diff --git a/contrib/meson/meson_options.txt b/contrib/meson/meson_options.txt index 1d28c71a8..6e9358576 100644 --- a/contrib/meson/meson_options.txt +++ b/contrib/meson/meson_options.txt @@ -9,7 +9,7 @@ # ############################################################################# option('multithread_support', type: 'boolean', value: true, - description: 'Enable multithreading when pthread is detected') + description: 'Enable multi-threading when pthread is detected') option('legacy_support', type: 'string', value: '4', description: 'Support any legacy format: true or false, or 7 to 1 for v0.7+ to v0.1+') option('build_programs', type: 'boolean', value: true, diff --git a/contrib/meson/programs/meson.build b/contrib/meson/programs/meson.build index 63ea328ae..8130ab152 100644 --- a/contrib/meson/programs/meson.build +++ b/contrib/meson/programs/meson.build @@ -61,8 +61,6 @@ executable('zstd-frugal', # Program symlinks # ============================================================================= -CreateSymlink_py = join_paths(meson.current_source_dir(), '..', 'CreateSymlink.py') - foreach f : [ 'zstdcat', 'unzstd' ] custom_target(f, output : f, @@ -89,7 +87,6 @@ endif zstd_man1_dir = join_paths(zstd_mandir, 'man1') zstd_1_file = join_paths(programs_dir, 'zstd.1') -CopyFile_py = join_paths(meson.current_source_dir(), '..', 'CopyFile.py') custom_target('zstd.1', output : 'zstd.1', From 8a160680d17b9719c4aea275af401a03ddfcb92d Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 11:49:52 +0700 Subject: [PATCH 14/62] Update legacy support to 5 --- contrib/meson/lib/meson.build | 9 ++++----- contrib/meson/meson_options.txt | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/contrib/meson/lib/meson.build b/contrib/meson/lib/meson.build index b5d6778dd..bc0220584 100644 --- a/contrib/meson/lib/meson.build +++ b/contrib/meson/lib/meson.build @@ -58,7 +58,7 @@ if legacy_support == '0' endif if legacy_support != 'false' if legacy_support == 'true' - legacy_support = '1' + legacy_support = '5' endif legacy_int = legacy_support.to_int() if legacy_int < 0 or legacy_int >= 8 @@ -108,12 +108,11 @@ libzstd = library('zstd', soversion: '1') pkgconfig.generate(name: 'libzstd', - description: 'fast lossless compression algorithm library', - version: zstd_version, filebase: 'libzstd', libraries: [libzstd], - #subdirs: ['.'] - ) + description: 'fast lossless compression algorithm library', + version: zstd_version, + url: 'http://www.zstd.net/') install_headers(join_paths(library_dir, 'zstd.h'), join_paths(library_deprecated_dir, 'zbuff.h'), diff --git a/contrib/meson/meson_options.txt b/contrib/meson/meson_options.txt index 6e9358576..066be6a32 100644 --- a/contrib/meson/meson_options.txt +++ b/contrib/meson/meson_options.txt @@ -10,7 +10,7 @@ option('multithread_support', type: 'boolean', value: true, description: 'Enable multi-threading when pthread is detected') -option('legacy_support', type: 'string', value: '4', +option('legacy_support', type: 'string', value: '5', description: 'Support any legacy format: true or false, or 7 to 1 for v0.7+ to v0.1+') option('build_programs', type: 'boolean', value: true, description: 'Enable programs build') From 5a36a57cf5b710d82679cffd64f51080c14fe4c0 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 11:57:53 +0700 Subject: [PATCH 15/62] Bump to 1.3.8 and fix run_command function The run_command is run from an unspecified directory. Therefore we cannot assume which directory it is running our command. --- contrib/meson/meson.build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index 48f8d570d..bd7eeff73 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -14,7 +14,7 @@ project('zstd', default_options : ['c_std=c99', 'cpp_std=c++11', 'buildtype=release'], - version: '1.3.7', + version: '1.3.8', # for install_man meson_version: '>=0.47.0') @@ -63,7 +63,7 @@ CopyFile_py = files('CopyFile.py') # Getting project version from zstd.h # ============================================================================= -zstd_h_file = join_paths(library_dir, 'zstd.h') +zstd_h_file = join_paths(meson.current_source_dir(), library_dir, 'zstd.h') r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file) if r.returncode() == 0 output = r.stdout().strip() From c9f01443027a40ff0153df94968d6f221d3efb2b Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 12:16:09 +0700 Subject: [PATCH 16/62] Fix meson tests build --- contrib/meson/meson.build | 1 + contrib/meson/tests/meson.build | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index bd7eeff73..1e0a2ec40 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -103,6 +103,7 @@ endif # ============================================================================= subdir('lib') + if enable_programs subdir('programs') endif diff --git a/contrib/meson/tests/meson.build b/contrib/meson/tests/meson.build index cc9e1ee10..c64ae7a7a 100644 --- a/contrib/meson/tests/meson.build +++ b/contrib/meson/tests/meson.build @@ -31,7 +31,7 @@ fullbench_sources = [datagen_c_file, util_c_file, benchfn_c_file, join_paths(programs_dir, 'benchzstd.c'), - join_paths(programs_dir, 'fullbench.c')] + join_paths(tests_dir, 'fullbench.c')] fullbench = executable('fullbench', fullbench_sources, include_directories: test_includes, @@ -49,10 +49,10 @@ fuzzer = executable('fuzzer', install: false) test('fuzzer', fuzzer) -paramgrill_sources = [benchfn_c_file +paramgrill_sources = [benchfn_c_file, join_paths(programs_dir, 'benchzstd.c'), - datagen_c_file - util_c_file + datagen_c_file, + util_c_file, join_paths(tests_dir, 'paramgrill.c')] if host_machine.system() != 'windows' paramgrill = executable('paramgrill', @@ -60,6 +60,6 @@ if host_machine.system() != 'windows' include_directories: test_includes, link_with: libzstd, dependencies: libm_dep, - install: false ) + install: false) test('paramgrill', paramgrill) endif From 337f914dc8e551916b164ddbb91dd77ac9afffbd Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 14:26:57 +0700 Subject: [PATCH 17/62] Fix lib soversion and no install cover.h header --- contrib/meson/lib/meson.build | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/contrib/meson/lib/meson.build b/contrib/meson/lib/meson.build index bc0220584..0c8a750a6 100644 --- a/contrib/meson/lib/meson.build +++ b/contrib/meson/lib/meson.build @@ -67,7 +67,7 @@ if legacy_support != 'false' add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_int), language: 'c') libzstd_includes += [ include_directories(library_legacy_dir) ] - # See ZSTD_LEGACY_SUPPORT of programs/README.md + # See ZSTD_LEGACY_SUPPORT of lib/README.md message('Enable legacy support back to version 0.@0@'.format(legacy_int)) if legacy_int <= 1 libzstd_sources += join_paths(library_legacy_dir, 'zstd_v01.c') @@ -105,7 +105,7 @@ libzstd = library('zstd', include_directories: libzstd_includes, dependencies: libzstd_deps, install: true, - soversion: '1') + soversion: zstd_version) pkgconfig.generate(name: 'libzstd', filebase: 'libzstd', @@ -117,5 +117,4 @@ pkgconfig.generate(name: 'libzstd', install_headers(join_paths(library_dir, 'zstd.h'), join_paths(library_deprecated_dir, 'zbuff.h'), join_paths(library_dictbuilder_dir, 'zdict.h'), - join_paths(library_dictbuilder_dir, 'cover.h'), join_paths(library_common_dir, 'zstd_errors.h')) From 0a25b13aae4c1c669e5ea7ced3fe687b1d255c71 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 13:06:18 +0700 Subject: [PATCH 18/62] Init CI test for meson build Do not use ccache: Build error in gcc-8 --- .travis.yml | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 226acaa37..777e5ad86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,11 +3,13 @@ language: c dist: trusty sudo: required + addons: apt: update: true matrix: + fast_finish: true include: # Ubuntu 14.04 - env: Cmd='make test' @@ -49,6 +51,10 @@ matrix: - if: tag =~ ^v[0-9]\.[0-9] env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH' + - env: INSTALL_SYSTEM='meson' + allow_failures: + - env: INSTALL_SYSTEM='meson' + git: depth: 1 @@ -59,9 +65,25 @@ branches: - travisTest script: - - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST - - export FUZZERTEST=-T2mn; - export ZSTREAM_TESTTIME=-T2mn; - export DECODECORPUS_TESTTIME=-T1mn; - sh -c "$Cmd" || travis_terminate 1; + - JOB_NUMBER=$(printf '%s' "${TRAVIS_JOB_NUMBER}" | sed -E 's@[0-9]+\.([0-9]+)@\1@') + - printf 'JOB_NUMBER=%s TRAVIS_BRANCH=%s TRAVIS_EVENT_TYPE=%s TRAVIS_PULL_REQUEST=%s\n' + "${JOB_NUMBER}" "${TRAVIS_BRANCH}" "${TRAVIS_EVENT_TYPE}" "${TRAVIS_PULL_REQUEST}" + - if [ "${INSTALL_SYSTEM}" = meson ]; then + sudo apt-get install -qq python3.5 wget tree + && wget https://bootstrap.pypa.io/get-pip.py + && python3.5 get-pip.py --user + && rm get-pip.py + && pip3.5 install --user meson ninja; + mkdir contrib/meson/build; + pushd "$_"; + meson --buildtype=debug -Dbuild_contrib=true -Dbuild_tests=true .. + && ninja + && DESTDIR=./staging ninja install + && tree ./staging; + popd; + else + export FUZZERTEST=-T2mn; + export ZSTREAM_TESTTIME=-T2mn; + export DECODECORPUS_TESTTIME=-T1mn; + sh -c "${Cmd}" || travis_terminate 1; + fi From 3175188407ea8d02587f4c3bc0137af22876d42d Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 21:59:35 +0700 Subject: [PATCH 19/62] No need these helpers --- contrib/meson/CopyFile.py | 35 --------------------------------- contrib/meson/CreateSymlink.py | 36 ---------------------------------- 2 files changed, 71 deletions(-) delete mode 100644 contrib/meson/CopyFile.py delete mode 100644 contrib/meson/CreateSymlink.py diff --git a/contrib/meson/CopyFile.py b/contrib/meson/CopyFile.py deleted file mode 100644 index 6c0288ee4..000000000 --- a/contrib/meson/CopyFile.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -# ############################################################################# -# Copyright (c) 2018-present lzutao -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ############################################################################# -import os -import sys -import shutil - - -def usage(): - print('usage: python3 CreateSymlink.py ') - print('Copy the file named src to a file named dst') - sys.exit(1) - - -def main(): - if len(sys.argv) < 3: - usage() - src = sys.argv[1] - dst = sys.argv[2] - - if os.path.exists(dst): - print ('File already exists: %r' % (dst)) - return - - shutil.copy2(src, dst) - - -if __name__ == '__main__': - main() diff --git a/contrib/meson/CreateSymlink.py b/contrib/meson/CreateSymlink.py deleted file mode 100644 index d0f9918ab..000000000 --- a/contrib/meson/CreateSymlink.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -# ############################################################################# -# Copyright (c) 2018-present lzutao -# All rights reserved. -# -# This source code is licensed under both the BSD-style license (found in the -# LICENSE file in the root directory of this source tree) and the GPLv2 (found -# in the COPYING file in the root directory of this source tree). -# ############################################################################# -import os -import sys - - -def usage(): - print('usage: python3 CreateSymlink.py [dst is dir: True or False]') - sys.exit(1) - - -def main(): - if len(sys.argv) < 3: - usage() - src = sys.argv[1] - dst = sys.argv[2] - is_dir = False - if len(sys.argv) == 4: - is_dir = bool(sys.argv[3]) - - if os.path.islink(dst) and os.readlink(dst) == src: - print ('File exists: %r -> %r' % (dst, src)) - return - - os.symlink(src, dst, is_dir) - - -if __name__ == '__main__': - main() From ef2e76193710797101c133ed73119d553448abcc Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 22:31:16 +0700 Subject: [PATCH 20/62] Helper script to install symlink in meson --- contrib/meson/InstallSymlink.py | 72 +++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 contrib/meson/InstallSymlink.py diff --git a/contrib/meson/InstallSymlink.py b/contrib/meson/InstallSymlink.py new file mode 100644 index 000000000..d7b1e5a0b --- /dev/null +++ b/contrib/meson/InstallSymlink.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# ############################################################################# +# Copyright (c) 2018-present lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# +import errno +import os + + +def mkdir_p(path, dir_mode=0o777): + try: + os.makedirs(path, mode=dir_mode) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + +def InstallSymlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777): + if not os.path.exists(install_dir): + mkdir_p(install_dir, dir_mode) + if not os.path.isdir(install_dir): + raise NotADirectoryError(install_dir) + + new_dst = os.path.join(install_dir, dst) + if os.path.islink(new_dst) and os.readlink(new_dst) == src: + print('File exists: %r -> %r' % (dst, src)) + return + print('Installing symlink %r -> %r' % (new_dst, src)) + os.symlink(src, new_dst, dst_is_dir) + + +def main(): + import argparse + parser = argparse.ArgumentParser(description='Install a symlink.\n', + usage='usage: InstallSymlink.py [-h] [-d] [-m MODE] src dst ' + 'install_dir\n\n' + 'example:\n' + '\tInstallSymlink.py libcrypto.so.1.0.0 libcrypt.so ' + '/usr/lib/x86_64-linux-gnu False') + parser.add_argument('src', help='target to link') + parser.add_argument('dst', help='link name') + parser.add_argument('install_dir', help='installation directory') + parser.add_argument('-d', '--isdir', + action='store_true', + help='dst is a directory') + parser.add_argument('-m', '--mode', + help='directory mode on creating if not exist', + default='0o777') + args = parser.parse_args() + + src = args.src + dst = args.dst + install_dir = args.install_dir + dst_is_dir = args.isdir + dir_mode = int(args.mode, 8) + + DESTDIR = os.environ.get('DESTDIR') + if DESTDIR: + install_dir = DESTDIR + install_dir if os.path.isabs(install_dir) \ + else os.path.join(DESTDIR, install_dir) + + InstallSymlink(src, dst, install_dir, dst_is_dir, dir_mode) + + +if __name__ == '__main__': + main() From d79df2a370897dea8dad7457f3e35fe92d003f6a Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 22:32:37 +0700 Subject: [PATCH 21/62] Apply new InstallSymlink script --- contrib/meson/meson.build | 2 -- contrib/meson/programs/meson.build | 45 ++++++------------------------ 2 files changed, 9 insertions(+), 38 deletions(-) diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index 1e0a2ec40..065c5e1b9 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -56,8 +56,6 @@ enable_lzma = get_option('lzma_support') # ============================================================================= GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py') -CreateSymlink_py = files('CreateSymlink.py') -CopyFile_py = files('CopyFile.py') # ============================================================================= # Getting project version from zstd.h diff --git a/contrib/meson/programs/meson.build b/contrib/meson/programs/meson.build index 8130ab152..b29c31217 100644 --- a/contrib/meson/programs/meson.build +++ b/contrib/meson/programs/meson.build @@ -61,24 +61,12 @@ executable('zstd-frugal', # Program symlinks # ============================================================================= -foreach f : [ 'zstdcat', 'unzstd' ] - custom_target(f, - output : f, - input: zstd, - command : [python3, CreateSymlink_py, '@PLAINNAME@', '@OUTPUT@'], - build_always_stale: false, - install : true, - install_dir: zstd_bindir) -endforeach +InstallSymlink_py = join_paths('..', 'InstallSymlink.py') +meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdcat', zstd_bindir) +meson.add_install_script(InstallSymlink_py, 'zstd', 'unzstd', zstd_bindir) if enable_multithread - custom_target('zstdmt', - output : 'zstdmt', - input: zstd, - command : [python3, CreateSymlink_py, '@PLAINNAME@', '@OUTPUT@'], - build_always_stale: false, - install : true, - install_dir: zstd_bindir) + meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdmt', zstd_bindir) endif # ============================================================================= @@ -86,25 +74,10 @@ endif # ============================================================================= zstd_man1_dir = join_paths(zstd_mandir, 'man1') -zstd_1_file = join_paths(programs_dir, 'zstd.1') -custom_target('zstd.1', - output : 'zstd.1', - input: zstd_1_file, - command : [python3, CopyFile_py, '@INPUT@', '@OUTPUT@'], - build_always_stale: false, - install : true, - install_dir: zstd_man1_dir) - -foreach f : [ 'zstdcat.1', 'unzstd.1' ] - custom_target(f, - output : f, - input: zstd_1_file, - command : [python3, CreateSymlink_py, '@PLAINNAME@', '@OUTPUT@'], - install : true, - build_always_stale: false, - install_dir: zstd_man1_dir) -endforeach - -install_man(join_paths(programs_dir, 'zstdgrep.1'), +install_man(join_paths(programs_dir, 'zstd.1'), + join_paths(programs_dir, 'zstdgrep.1'), join_paths(programs_dir, 'zstdless.1')) + +meson.add_install_script(InstallSymlink_py, 'zstd.1', 'zstdcat.1', zstd_man1_dir) +meson.add_install_script(InstallSymlink_py, 'zstd.1', 'unzstd.1', zstd_man1_dir) From 9c862c6a53fe60aaa5ba6424081e8d07c66460be Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Wed, 28 Nov 2018 23:59:42 +0700 Subject: [PATCH 22/62] Fix manpage symlinks [skip ci] --- contrib/meson/contrib/gen_html/meson.build | 8 +++----- contrib/meson/programs/meson.build | 5 +++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/contrib/meson/contrib/gen_html/meson.build b/contrib/meson/contrib/gen_html/meson.build index 38ed52761..d9185dfe4 100644 --- a/contrib/meson/contrib/gen_html/meson.build +++ b/contrib/meson/contrib/gen_html/meson.build @@ -21,16 +21,14 @@ gen_html_includes = include_directories(programs_dir, gen_html = executable('gen_html', join_paths(contrib_gen_html_dir, 'gen_html.cpp'), include_directories: gen_html_includes, - install: false ) + install: false) # Update zstd manual zstd_manual_html = custom_target('zstd_manual.html', output : 'zstd_manual.html', - command : [ - gen_html, + command : [gen_html, zstd_version, join_paths(meson.current_source_dir(), library_dir, 'zstd.h'), - '@OUTPUT@' - ], + '@OUTPUT@'], install : true, install_dir : zstd_docdir ) diff --git a/contrib/meson/programs/meson.build b/contrib/meson/programs/meson.build index b29c31217..1c77f7480 100644 --- a/contrib/meson/programs/meson.build +++ b/contrib/meson/programs/meson.build @@ -79,5 +79,6 @@ install_man(join_paths(programs_dir, 'zstd.1'), join_paths(programs_dir, 'zstdgrep.1'), join_paths(programs_dir, 'zstdless.1')) -meson.add_install_script(InstallSymlink_py, 'zstd.1', 'zstdcat.1', zstd_man1_dir) -meson.add_install_script(InstallSymlink_py, 'zstd.1', 'unzstd.1', zstd_man1_dir) +# Meson automatically compresses manpages to gz format +meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdcat.1.gz', zstd_man1_dir) +meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'unzstd.1.gz', zstd_man1_dir) From 1985e427c70d1fa9454d83380d6427594ee5e180 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 00:09:24 +0700 Subject: [PATCH 23/62] Add manpage install warning [skip ci] We link new manpages with gz compressed format of the target manpage. I have not tested it on Windows. So just place a warning here. --- contrib/meson/programs/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/meson/programs/meson.build b/contrib/meson/programs/meson.build index 1c77f7480..59cf09a05 100644 --- a/contrib/meson/programs/meson.build +++ b/contrib/meson/programs/meson.build @@ -80,5 +80,6 @@ install_man(join_paths(programs_dir, 'zstd.1'), join_paths(programs_dir, 'zstdless.1')) # Meson automatically compresses manpages to gz format +# WARNING: This may fail on Windows. Test NEEDED. meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdcat.1.gz', zstd_man1_dir) meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'unzstd.1.gz', zstd_man1_dir) From d3134a3ed3674088e952813945bf1b7c9c4b73f8 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 01:26:41 +0700 Subject: [PATCH 24/62] Rename meson variables --- contrib/meson/meson.build | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index 065c5e1b9..a8df86b85 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -86,14 +86,14 @@ lzma_dep = dependency('lzma', required: false) if cxx.get_id() == 'gcc' or cxx.get_id() == 'clang' common_flags = [ '-DXXH_NAMESPACE=ZSTD_' ] - zstd_compilation_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ] - cc_common_flags = cc.get_supported_arguments(zstd_compilation_flags) - cc_common_flags += cc.get_supported_arguments(['-Wstrict-prototypes']) - cc_common_flags += common_flags - cxx_common_flags = cxx.get_supported_arguments(zstd_compilation_flags) - cxx_common_flags += common_flags - add_project_arguments(cc_common_flags, language : 'c') - add_project_arguments(cxx_common_flags, language : 'cpp') + common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ] + cc_compilation_flags = cc.get_supported_arguments(common_warning_flags) + cc_compilation_flags += cc.get_supported_arguments(['-Wstrict-prototypes']) + cc_compilation_flags += common_flags + cxx_compilation_flags = cxx.get_supported_arguments(common_warning_flags) + cxx_compilation_flags += common_flags + add_project_arguments(cc_compilation_flags, language : 'c') + add_project_arguments(cxx_compilation_flags, language : 'cpp') endif # ============================================================================= From 3f27e2a072638bf74d986644087d11c0a7c541cc Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 01:27:20 +0700 Subject: [PATCH 25/62] Install zstdmt.1 manpage [skip ci] --- contrib/meson/programs/meson.build | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/meson/programs/meson.build b/contrib/meson/programs/meson.build index 59cf09a05..ba47e0f97 100644 --- a/contrib/meson/programs/meson.build +++ b/contrib/meson/programs/meson.build @@ -83,3 +83,7 @@ install_man(join_paths(programs_dir, 'zstd.1'), # WARNING: This may fail on Windows. Test NEEDED. meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdcat.1.gz', zstd_man1_dir) meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'unzstd.1.gz', zstd_man1_dir) + +if enable_multithread + meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdmt.1.gz', zstd_man1_dir) +endif From f660825d9fd45050e9abb82ec99a53c0c7aacf78 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 01:50:24 +0700 Subject: [PATCH 26/62] Install missed zstdgrep and zstdless --- contrib/meson/programs/meson.build | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/meson/programs/meson.build b/contrib/meson/programs/meson.build index ba47e0f97..e9f1c518d 100644 --- a/contrib/meson/programs/meson.build +++ b/contrib/meson/programs/meson.build @@ -57,6 +57,10 @@ executable('zstd-frugal', c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ], install: true) +install_data(join_paths(programs_dir, 'zstdgrep'), + join_paths(programs_dir, 'zstdless'), + install_dir: zstd_bindir) + # ============================================================================= # Program symlinks # ============================================================================= From 6f3f1a8d3a5398c1f403d25659baa679181dbe80 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 10:54:01 +0700 Subject: [PATCH 27/62] No install zstd_manual.html --- contrib/meson/contrib/gen_html/meson.build | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/meson/contrib/gen_html/meson.build b/contrib/meson/contrib/gen_html/meson.build index d9185dfe4..9f02baf6e 100644 --- a/contrib/meson/contrib/gen_html/meson.build +++ b/contrib/meson/contrib/gen_html/meson.build @@ -30,5 +30,4 @@ zstd_manual_html = custom_target('zstd_manual.html', zstd_version, join_paths(meson.current_source_dir(), library_dir, 'zstd.h'), '@OUTPUT@'], - install : true, - install_dir : zstd_docdir ) + install : false) From 5c4965c351f3acfaebbe1297fddd729bd533997b Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 10:54:29 +0700 Subject: [PATCH 28/62] Add pedantic flag --- contrib/meson/contrib/pzstd/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/meson/contrib/pzstd/meson.build b/contrib/meson/contrib/pzstd/meson.build index 923990145..3531ace7d 100644 --- a/contrib/meson/contrib/pzstd/meson.build +++ b/contrib/meson/contrib/pzstd/meson.build @@ -25,7 +25,7 @@ pzstd_sources = [join_paths(programs_dir, 'util.c'), pzstd = executable('pzstd', pzstd_sources, - cpp_args: [ '-DNDEBUG', '-Wno-shadow' ], + cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ], include_directories: pzstd_includes, link_with: libzstd, dependencies: [ thread_dep ], From c0e71cae554bf20054eb5623c33154c2002a1bfd Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 11:05:40 +0700 Subject: [PATCH 29/62] Add enable_lz4 build option and fix lzma dependency --- contrib/meson/meson.build | 4 +++- contrib/meson/meson_options.txt | 2 ++ contrib/meson/programs/meson.build | 5 +++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index a8df86b85..966c5008e 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -50,6 +50,7 @@ enable_contrib = get_option('build_contrib') enable_multithread = get_option('multithread_support') enable_zlib = get_option('zlib_support') enable_lzma = get_option('lzma_support') +enable_lz4 = get_option('lz4_support') # ============================================================================= # Helper scripts for Meson @@ -78,7 +79,8 @@ endif libm_dep = cc.find_library('m', required: true) thread_dep = dependency('threads', required: false) zlib_dep = dependency('zlib', required: false) -lzma_dep = dependency('lzma', required: false) +lzma_dep = dependency('liblzma', required: false) +lz4_dep = dependency('liblz4', required: false) # ============================================================================= # Compiler flags diff --git a/contrib/meson/meson_options.txt b/contrib/meson/meson_options.txt index 066be6a32..86a36523a 100644 --- a/contrib/meson/meson_options.txt +++ b/contrib/meson/meson_options.txt @@ -24,3 +24,5 @@ option('zlib_support', type: 'boolean', value: false, description: 'Enable zlib support') option('lzma_support', type: 'boolean', value: false, description: 'Enable lzma support') +option('lz4_support', type: 'boolean', value: false, + description: 'Enable lz4 support') diff --git a/contrib/meson/programs/meson.build b/contrib/meson/programs/meson.build index e9f1c518d..c96b6d458 100644 --- a/contrib/meson/programs/meson.build +++ b/contrib/meson/programs/meson.build @@ -38,6 +38,11 @@ if enable_lzma and lzma_dep.found() zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ] endif +if enable_lz4 and lz4_dep.found() + zstd_deps += [ lz4_dep ] + zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ] +endif + zstd = executable('zstd', zstd_programs_sources, c_args: zstd_c_args, From beb13bd87ebea7e3f7bafaed8696e91f5359c649 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 11:45:57 +0700 Subject: [PATCH 30/62] Move contrib/meson to build/meson --- {contrib => build}/meson/GetZstdLibraryVersion.py | 0 {contrib => build}/meson/InstallSymlink.py | 0 {contrib => build}/meson/README.md | 0 {contrib => build}/meson/contrib/gen_html/meson.build | 0 {contrib => build}/meson/contrib/meson.build | 0 {contrib => build}/meson/contrib/pzstd/meson.build | 0 {contrib => build}/meson/lib/meson.build | 0 {contrib => build}/meson/meson.build | 0 {contrib => build}/meson/meson_options.txt | 0 {contrib => build}/meson/programs/meson.build | 0 {contrib => build}/meson/tests/meson.build | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename {contrib => build}/meson/GetZstdLibraryVersion.py (100%) rename {contrib => build}/meson/InstallSymlink.py (100%) rename {contrib => build}/meson/README.md (100%) rename {contrib => build}/meson/contrib/gen_html/meson.build (100%) rename {contrib => build}/meson/contrib/meson.build (100%) rename {contrib => build}/meson/contrib/pzstd/meson.build (100%) rename {contrib => build}/meson/lib/meson.build (100%) rename {contrib => build}/meson/meson.build (100%) rename {contrib => build}/meson/meson_options.txt (100%) rename {contrib => build}/meson/programs/meson.build (100%) rename {contrib => build}/meson/tests/meson.build (100%) diff --git a/contrib/meson/GetZstdLibraryVersion.py b/build/meson/GetZstdLibraryVersion.py similarity index 100% rename from contrib/meson/GetZstdLibraryVersion.py rename to build/meson/GetZstdLibraryVersion.py diff --git a/contrib/meson/InstallSymlink.py b/build/meson/InstallSymlink.py similarity index 100% rename from contrib/meson/InstallSymlink.py rename to build/meson/InstallSymlink.py diff --git a/contrib/meson/README.md b/build/meson/README.md similarity index 100% rename from contrib/meson/README.md rename to build/meson/README.md diff --git a/contrib/meson/contrib/gen_html/meson.build b/build/meson/contrib/gen_html/meson.build similarity index 100% rename from contrib/meson/contrib/gen_html/meson.build rename to build/meson/contrib/gen_html/meson.build diff --git a/contrib/meson/contrib/meson.build b/build/meson/contrib/meson.build similarity index 100% rename from contrib/meson/contrib/meson.build rename to build/meson/contrib/meson.build diff --git a/contrib/meson/contrib/pzstd/meson.build b/build/meson/contrib/pzstd/meson.build similarity index 100% rename from contrib/meson/contrib/pzstd/meson.build rename to build/meson/contrib/pzstd/meson.build diff --git a/contrib/meson/lib/meson.build b/build/meson/lib/meson.build similarity index 100% rename from contrib/meson/lib/meson.build rename to build/meson/lib/meson.build diff --git a/contrib/meson/meson.build b/build/meson/meson.build similarity index 100% rename from contrib/meson/meson.build rename to build/meson/meson.build diff --git a/contrib/meson/meson_options.txt b/build/meson/meson_options.txt similarity index 100% rename from contrib/meson/meson_options.txt rename to build/meson/meson_options.txt diff --git a/contrib/meson/programs/meson.build b/build/meson/programs/meson.build similarity index 100% rename from contrib/meson/programs/meson.build rename to build/meson/programs/meson.build diff --git a/contrib/meson/tests/meson.build b/build/meson/tests/meson.build similarity index 100% rename from contrib/meson/tests/meson.build rename to build/meson/tests/meson.build From 0669392937c5161d52247c118687318daf09f8e1 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 11:49:33 +0700 Subject: [PATCH 31/62] Update relevant files after moving meson to build/ Update meson build instructions --- .travis.yml | 2 +- README.md | 2 +- build/meson/README.md | 19 +++++++++++++------ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 777e5ad86..dbea289f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,7 +74,7 @@ script: && python3.5 get-pip.py --user && rm get-pip.py && pip3.5 install --user meson ninja; - mkdir contrib/meson/build; + mkdir build/meson/build; pushd "$_"; meson --buildtype=debug -Dbuild_contrib=true -Dbuild_tests=true .. && ninja diff --git a/README.md b/README.md index dc99dc0fd..c0b550572 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ By default, `CMAKE_BUILD_TYPE` is set to `Release`. #### Meson -A Meson project is provided within `contrib/meson`. +A Meson project is provided within `build/meson`. #### Visual Studio (Windows) diff --git a/build/meson/README.md b/build/meson/README.md index a2730faaa..f5818cc69 100644 --- a/build/meson/README.md +++ b/build/meson/README.md @@ -1,16 +1,23 @@ -This Meson project is provided with no guarantee and maintained +contrib/meson - Meson build system for zstandard +================================================ + +Meson is a build system designed to optimize programmer productivity. +It aims to do this by providing simple, out-of-the-box support for +modern software development tools and practices, such as unit tests, +coverage reports, Valgrind, CCache and the like. + +This Meson build system is provided with no guarantee and maintained by Dima Krasner . It outputs one `libzstd`, either shared or static, depending on `default_library` option. -How to build -============ +## How to build -`cd` to this meson directory (`zstd/contrib/meson`) and type: +`cd` to this meson directory (`build/meson`) ```sh -meson --buildtype=release --strip --prefix=/usr builddir +meson --buildtype=release -D with-contrib=true -D with-tests=true -D with-contrib=true builddir cd builddir ninja # to build ninja install # to install @@ -22,7 +29,7 @@ You might want to install it in staging directory: DESTDIR=./staging ninja install ``` -To configure the build, use: +To configure build options, use: ```sh meson configure From 9ac9cbc9de892c5d960858a1249a99e1f39ed104 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 12:24:43 +0700 Subject: [PATCH 32/62] Enable debugging flag --- build/meson/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/meson/meson.build b/build/meson/meson.build index 966c5008e..18f44e0af 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -42,7 +42,7 @@ contrib_meson_dir = join_paths(zstd_source_dir, 'contrib', 'meson') # ============================================================================= # Project options # ============================================================================= - +enable_debug = get_option('debug') legacy_support = get_option('legacy_support') enable_programs = get_option('build_programs') enable_tests = get_option('build_tests') From 918e0d5c3668937ce10b7a971fe383b6eb2c03ad Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 12:52:35 +0700 Subject: [PATCH 33/62] Rename some meson build options to match autoconf convention --- .travis.yml | 2 +- build/meson/lib/meson.build | 12 ++++++------ build/meson/meson.build | 23 ++++++++++++----------- build/meson/meson_options.txt | 18 +++++++++--------- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index dbea289f6..54d8e431a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,7 +76,7 @@ script: && pip3.5 install --user meson ninja; mkdir build/meson/build; pushd "$_"; - meson --buildtype=debug -Dbuild_contrib=true -Dbuild_tests=true .. + meson --buildtype=debug -D with-contrib=true -D with-tests=true -D with-contrib=true .. && ninja && DESTDIR=./staging ninja install && tree ./staging; diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 0c8a750a6..8a693dc71 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -53,14 +53,14 @@ libzstd_sources = [join_paths(library_common_dir, 'entropy_common.c'), join_paths(library_deprecated_dir, 'zbuff_compress.c'), join_paths(library_deprecated_dir, 'zbuff_decompress.c')] -if legacy_support == '0' - legacy_support = 'false' +if with_legacy_support == '0' + with_legacy_support = 'false' endif -if legacy_support != 'false' - if legacy_support == 'true' - legacy_support = '5' +if with_legacy_support != 'false' + if with_legacy_support == 'true' + with_legacy_support = '5' endif - legacy_int = legacy_support.to_int() + legacy_int = with_legacy_support.to_int() if legacy_int < 0 or legacy_int >= 8 legacy_int = 0 endif diff --git a/build/meson/meson.build b/build/meson/meson.build index 18f44e0af..7b9b2520e 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -42,15 +42,16 @@ contrib_meson_dir = join_paths(zstd_source_dir, 'contrib', 'meson') # ============================================================================= # Project options # ============================================================================= + enable_debug = get_option('debug') -legacy_support = get_option('legacy_support') -enable_programs = get_option('build_programs') -enable_tests = get_option('build_tests') -enable_contrib = get_option('build_contrib') -enable_multithread = get_option('multithread_support') -enable_zlib = get_option('zlib_support') -enable_lzma = get_option('lzma_support') -enable_lz4 = get_option('lz4_support') +with_legacy_support = get_option('with-legacy-support') +with_programs = get_option('with-programs') +with_tests = get_option('with-tests') +with_contrib = get_option('with-contrib') +enable_multithread = get_option('enable-multithread') +enable_zlib = get_option('enable-zlib') +enable_lzma = get_option('enable-lzma') +enable_lz4 = get_option('enable-lz4') # ============================================================================= # Helper scripts for Meson @@ -104,14 +105,14 @@ endif subdir('lib') -if enable_programs +if with_programs subdir('programs') endif -if enable_tests +if with_tests subdir('tests') endif -if enable_contrib +if with_contrib subdir('contrib') endif diff --git a/build/meson/meson_options.txt b/build/meson/meson_options.txt index 86a36523a..dca6df809 100644 --- a/build/meson/meson_options.txt +++ b/build/meson/meson_options.txt @@ -8,21 +8,21 @@ # in the COPYING file in the root directory of this source tree). # ############################################################################# -option('multithread_support', type: 'boolean', value: true, +option('enable-multithread', type: 'boolean', value: true, description: 'Enable multi-threading when pthread is detected') -option('legacy_support', type: 'string', value: '5', +option('with-legacy-support', type: 'string', value: '5', description: 'Support any legacy format: true or false, or 7 to 1 for v0.7+ to v0.1+') -option('build_programs', type: 'boolean', value: true, +option('with-programs', type: 'boolean', value: true, description: 'Enable programs build') -option('build_contrib', type: 'boolean', value: false, +option('with-contrib', type: 'boolean', value: false, description: 'Enable contrib build') -option('build_tests', type: 'boolean', value: false, +option('with-tests', type: 'boolean', value: false, description: 'Enable tests build') -option('use_static_runtime', type: 'boolean', value: false, +option('enable-static-runtime', type: 'boolean', value: false, description: 'Link to static run-time libraries on MSVC') -option('zlib_support', type: 'boolean', value: false, +option('enable-zlib', type: 'boolean', value: false, description: 'Enable zlib support') -option('lzma_support', type: 'boolean', value: false, +option('enable-lzma', type: 'boolean', value: false, description: 'Enable lzma support') -option('lz4_support', type: 'boolean', value: false, +option('enable-lz4', type: 'boolean', value: false, description: 'Enable lz4 support') From c4fb45ffe8c8dae1205de108b2e8727d0b1ffcf7 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 13:15:18 +0700 Subject: [PATCH 34/62] Add comment about looking for dependencies --- build/meson/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/build/meson/meson.build b/build/meson/meson.build index 7b9b2520e..750e841cf 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -79,6 +79,7 @@ endif libm_dep = cc.find_library('m', required: true) thread_dep = dependency('threads', required: false) +# Arguments in dependency should be equivalent to those passed to pkg-config zlib_dep = dependency('zlib', required: false) lzma_dep = dependency('liblzma', required: false) lz4_dep = dependency('liblz4', required: false) From 8c59d94d7a5b23dff403628392d0b9fe5f388ca0 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Thu, 29 Nov 2018 17:48:18 +0700 Subject: [PATCH 35/62] Build both shared and static lib on CI Rename INSTALL_SYSTEM to BUILD_SYSTEM --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 54d8e431a..8a0170e04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,9 +51,9 @@ matrix: - if: tag =~ ^v[0-9]\.[0-9] env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH' - - env: INSTALL_SYSTEM='meson' + - env: BUILD_SYSTEM='meson' allow_failures: - - env: INSTALL_SYSTEM='meson' + - env: BUILD_SYSTEM='meson' git: depth: 1 @@ -68,7 +68,7 @@ script: - JOB_NUMBER=$(printf '%s' "${TRAVIS_JOB_NUMBER}" | sed -E 's@[0-9]+\.([0-9]+)@\1@') - printf 'JOB_NUMBER=%s TRAVIS_BRANCH=%s TRAVIS_EVENT_TYPE=%s TRAVIS_PULL_REQUEST=%s\n' "${JOB_NUMBER}" "${TRAVIS_BRANCH}" "${TRAVIS_EVENT_TYPE}" "${TRAVIS_PULL_REQUEST}" - - if [ "${INSTALL_SYSTEM}" = meson ]; then + - if [ "${BUILD_SYSTEM}" = meson ]; then sudo apt-get install -qq python3.5 wget tree && wget https://bootstrap.pypa.io/get-pip.py && python3.5 get-pip.py --user @@ -76,7 +76,8 @@ script: && pip3.5 install --user meson ninja; mkdir build/meson/build; pushd "$_"; - meson --buildtype=debug -D with-contrib=true -D with-tests=true -D with-contrib=true .. + meson --buildtype=debug -D with-contrib=true -D with-tests=true + -D with-contrib=true -D default_library=both .. && ninja && DESTDIR=./staging ninja install && tree ./staging; From 2337429e8d270a1d25c10467e6246ce9133478eb Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 30 Nov 2018 11:01:19 +0700 Subject: [PATCH 36/62] Change tabsize --- build/meson/GetZstdLibraryVersion.py | 34 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/build/meson/GetZstdLibraryVersion.py b/build/meson/GetZstdLibraryVersion.py index 364edc5aa..5659a3283 100644 --- a/build/meson/GetZstdLibraryVersion.py +++ b/build/meson/GetZstdLibraryVersion.py @@ -12,34 +12,34 @@ import sys def usage(): - print('usage: python3 GetZstdLibraryVersion.py ') - sys.exit(1) + print('usage: python3 GetZstdLibraryVersion.py ') + sys.exit(1) def find_version(filepath): - version_file_data = None - with open(filepath) as fd: - version_file_data = fd.read() + version_file_data = None + with open(filepath) as fd: + version_file_data = fd.read() - patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+) + patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+) #\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+) #\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+) """ - regex = re.compile(patterns, re.MULTILINE) - version_match = regex.search(version_file_data) - if version_match: - return version_match.groups() - raise RuntimeError("Unable to find version string.") + regex = re.compile(patterns, re.MULTILINE) + version_match = regex.search(version_file_data) + if version_match: + return version_match.groups() + raise RuntimeError("Unable to find version string.") def main(): - if len(sys.argv) < 2: - usage() + if len(sys.argv) < 2: + usage() - filepath = sys.argv[1] - version_tup = find_version(filepath) - print('.'.join(version_tup)) + filepath = sys.argv[1] + version_tup = find_version(filepath) + print('.'.join(version_tup)) if __name__ == '__main__': - main() + main() From 0a0683f5b2a0c0ed1ecbe6d27f9717c1ca176c5a Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 30 Nov 2018 15:30:45 +0700 Subject: [PATCH 37/62] Initial support for Windows build --- build/meson/contrib/gen_html/meson.build | 32 ++-- build/meson/contrib/pzstd/meson.build | 34 ++-- build/meson/lib/meson.build | 206 ++++++++++++----------- build/meson/meson.build | 88 +++++++--- build/meson/meson_options.txt | 26 +-- build/meson/programs/meson.build | 100 ++++++----- 6 files changed, 269 insertions(+), 217 deletions(-) diff --git a/build/meson/contrib/gen_html/meson.build b/build/meson/contrib/gen_html/meson.build index 9f02baf6e..a2fea9d52 100644 --- a/build/meson/contrib/gen_html/meson.build +++ b/build/meson/contrib/gen_html/meson.build @@ -7,27 +7,23 @@ # in the COPYING file in the root directory of this source tree). # ############################################################################# -zstd_source_dir = join_paths('..', '..', '..', '..') -library_dir = join_paths(zstd_source_dir, 'lib') -library_common_dir = join_paths(library_dir, 'common') -programs_dir = join_paths(zstd_source_dir, 'programs') -contrib_gen_html_dir = join_paths(zstd_source_dir, 'contrib', 'gen_html') +zstd_rootdir = '../../../..' -gen_html_includes = include_directories(programs_dir, - library_dir, - library_common_dir, - contrib_gen_html_dir) +gen_html_includes = include_directories(join_paths(zstd_rootdir, 'programs'), + join_paths(zstd_rootdir, 'lib'), + join_paths(zstd_rootdir, 'lib/common'), + join_paths(zstd_rootdir, 'contrib/gen_html')) gen_html = executable('gen_html', - join_paths(contrib_gen_html_dir, 'gen_html.cpp'), - include_directories: gen_html_includes, - install: false) + join_paths(zstd_rootdir, 'contrib/gen_html/gen_html.cpp'), + include_directories: gen_html_includes, + install: false) # Update zstd manual zstd_manual_html = custom_target('zstd_manual.html', - output : 'zstd_manual.html', - command : [gen_html, - zstd_version, - join_paths(meson.current_source_dir(), library_dir, 'zstd.h'), - '@OUTPUT@'], - install : false) + output : 'zstd_manual.html', + command : [gen_html, + zstd_version, + join_paths(meson.current_source_dir(), join_paths(zstd_rootdir, 'lib/zstd.h')), + '@OUTPUT@'], + install : false) diff --git a/build/meson/contrib/pzstd/meson.build b/build/meson/contrib/pzstd/meson.build index 3531ace7d..8f3822fd7 100644 --- a/build/meson/contrib/pzstd/meson.build +++ b/build/meson/contrib/pzstd/meson.build @@ -7,26 +7,18 @@ # in the COPYING file in the root directory of this source tree). # ############################################################################# -zstd_source_dir = join_paths('..', '..', '..', '..') -library_dir = join_paths(zstd_source_dir, 'lib') -library_common_dir = join_paths(library_dir, 'common') -programs_dir = join_paths(zstd_source_dir, 'programs') -contrib_pzstd_dir = join_paths(zstd_source_dir, 'contrib', 'pzstd') - -pzstd_includes = include_directories(programs_dir, - library_dir, - library_common_dir, - contrib_pzstd_dir) -pzstd_sources = [join_paths(programs_dir, 'util.c'), - join_paths(contrib_pzstd_dir, 'main.cpp'), - join_paths(contrib_pzstd_dir, 'Options.cpp'), - join_paths(contrib_pzstd_dir, 'Pzstd.cpp'), - join_paths(contrib_pzstd_dir, 'SkippableFrame.cpp')] +zstd_rootdir = '../../../..' +pzstd_includes = include_directories(join_paths(zstd_rootdir, 'programs'), + join_paths(zstd_rootdir, 'contrib/pzstd')) +pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'contrib/pzstd/main.cpp'), + join_paths(zstd_rootdir, 'contrib/pzstd/Options.cpp'), + join_paths(zstd_rootdir, 'contrib/pzstd/Pzstd.cpp'), + join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')] pzstd = executable('pzstd', - pzstd_sources, - cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ], - include_directories: pzstd_includes, - link_with: libzstd, - dependencies: [ thread_dep ], - install: true) + pzstd_sources, + cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ], + include_directories: pzstd_includes, + dependencies: [ libzstd_dep, thread_dep ], + install: true) diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 8a693dc71..5bfdbb074 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -8,113 +8,121 @@ # in the COPYING file in the root directory of this source tree). # ############################################################################# -zstd_source_dir = join_paths('..', '..', '..') -library_dir = join_paths(zstd_source_dir, 'lib') -library_common_dir = join_paths(library_dir, 'common') -library_compress_dir = join_paths(library_dir, 'compress') -library_decompress_dir = join_paths(library_dir, 'decompress') -library_dictbuilder_dir = join_paths(library_dir, 'dictBuilder') -library_deprecated_dir = join_paths(library_dir, 'deprecated') -library_legacy_dir = join_paths(library_dir, 'legacy') +zstd_rootdir = '../../..' -libzstd_includes = [include_directories(library_dir, - library_common_dir, - library_compress_dir, - library_decompress_dir, - library_dictbuilder_dir, - library_deprecated_dir)] +libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'), + join_paths(zstd_rootdir, 'lib/common'), + join_paths(zstd_rootdir, 'lib/compress'), + join_paths(zstd_rootdir, 'lib/decompress'), + join_paths(zstd_rootdir, 'lib/dictBuilder'), + join_paths(zstd_rootdir, 'lib/deprecated'))] -libzstd_sources = [join_paths(library_common_dir, 'entropy_common.c'), - join_paths(library_common_dir, 'fse_decompress.c'), - join_paths(library_common_dir, 'threading.c'), - join_paths(library_common_dir, 'pool.c'), - join_paths(library_common_dir, 'zstd_common.c'), - join_paths(library_common_dir, 'error_private.c'), - join_paths(library_common_dir, 'xxhash.c'), - join_paths(library_compress_dir, 'hist.c'), - join_paths(library_compress_dir, 'fse_compress.c'), - join_paths(library_compress_dir, 'huf_compress.c'), - join_paths(library_compress_dir, 'zstd_compress.c'), - join_paths(library_compress_dir, 'zstdmt_compress.c'), - join_paths(library_compress_dir, 'zstd_fast.c'), - join_paths(library_compress_dir, 'zstd_double_fast.c'), - join_paths(library_compress_dir, 'zstd_lazy.c'), - join_paths(library_compress_dir, 'zstd_opt.c'), - join_paths(library_compress_dir, 'zstd_ldm.c'), - join_paths(library_decompress_dir, 'huf_decompress.c'), - join_paths(library_decompress_dir, 'zstd_decompress.c'), - join_paths(library_decompress_dir, 'zstd_decompress_block.c'), - join_paths(library_decompress_dir, 'zstd_ddict.c'), - join_paths(library_dictbuilder_dir, 'cover.c'), - join_paths(library_dictbuilder_dir, 'fastcover.c'), - join_paths(library_dictbuilder_dir, 'divsufsort.c'), - join_paths(library_dictbuilder_dir, 'zdict.c'), - join_paths(library_deprecated_dir, 'zbuff_common.c'), - join_paths(library_deprecated_dir, 'zbuff_compress.c'), - join_paths(library_deprecated_dir, 'zbuff_decompress.c')] +libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), + join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'), + join_paths(zstd_rootdir, 'lib/common/threading.c'), + join_paths(zstd_rootdir, 'lib/common/pool.c'), + join_paths(zstd_rootdir, 'lib/common/zstd_common.c'), + join_paths(zstd_rootdir, 'lib/common/error_private.c'), + join_paths(zstd_rootdir, 'lib/common/xxhash.c'), + join_paths(zstd_rootdir, 'lib/compress/hist.c'), + join_paths(zstd_rootdir, 'lib/compress/fse_compress.c'), + join_paths(zstd_rootdir, 'lib/compress/huf_compress.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'), + join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_lazy.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_opt.c'), + join_paths(zstd_rootdir, 'lib/compress/zstd_ldm.c'), + join_paths(zstd_rootdir, 'lib/decompress/huf_decompress.c'), + join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress.c'), + join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress_block.c'), + join_paths(zstd_rootdir, 'lib/decompress/zstd_ddict.c'), + join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'), + join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'), + join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'), + join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'), + join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'), + join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'), + join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')] -if with_legacy_support == '0' - with_legacy_support = 'false' -endif -if with_legacy_support != 'false' - if with_legacy_support == 'true' - with_legacy_support = '5' - endif - legacy_int = with_legacy_support.to_int() - if legacy_int < 0 or legacy_int >= 8 - legacy_int = 0 - endif - add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_int), - language: 'c') - libzstd_includes += [ include_directories(library_legacy_dir) ] - # See ZSTD_LEGACY_SUPPORT of lib/README.md - message('Enable legacy support back to version 0.@0@'.format(legacy_int)) - if legacy_int <= 1 - libzstd_sources += join_paths(library_legacy_dir, 'zstd_v01.c') - endif - if legacy_int <= 2 - libzstd_sources += join_paths(library_legacy_dir, 'zstd_v02.c') - endif - if legacy_int <= 3 - libzstd_sources += join_paths(library_legacy_dir, 'zstd_v03.c') - endif - if legacy_int <= 4 - libzstd_sources += join_paths(library_legacy_dir, 'zstd_v04.c') - endif - if legacy_int <= 5 - libzstd_sources += join_paths(library_legacy_dir, 'zstd_v05.c') - endif - if legacy_int <= 6 - libzstd_sources += join_paths(library_legacy_dir, 'zstd_v06.c') - endif - if legacy_int <= 7 - libzstd_sources += join_paths(library_legacy_dir, 'zstd_v07.c') - endif -endif +# Explicit define legacy support +add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(with_legacy_support), + language: 'c') -if enable_multithread - message('Enable multi-threading support') - add_project_arguments('-DZSTD_MULTITHREAD', language: 'c') - libzstd_deps = [ thread_dep ] +if with_legacy_support == 0 + message('Legacy support: DISABLED') else - libzstd_deps = [] + # See ZSTD_LEGACY_SUPPORT of lib/README.md + message('Enable legacy support back to version 0.@0@'.format(with_legacy_support)) + + libzstd_includes += [ include_directories(join_paths(zstd_rootdir, 'lib/legacy')) ] + foreach i : [1, 2, 3, 4, 5, 6, 7] + if with_legacy_support <= i + libzstd_sources += join_paths(zstd_rootdir, 'lib/legacy/zstd_v0@0@.c'.format(i)) + endif + endforeach endif +libzstd_deps = [] +if enable_multithread + message('Enable multi-threading support') + add_project_arguments('-DZSTD_MULTITHREAD', language: 'c') + libzstd_deps = [ thread_dep ] +endif + +libzstd_c_args = [] +if cc_id == compiler_msvc + if default_library_type != 'static' + libzstd_sources += [windows_mod.compile_resources( + join_paths(zstd_rootdir, 'build/VS2010/libzstd-dll/libzstd-dll.rc'))] + libzstd_c_args += ['-DZSTD_DLL_EXPORT=1', + '-DZSTD_HEAPMODE=0', + '-D_CONSOLE', + '-D_CRT_SECURE_NO_WARNINGS'] + else + libzstd_c_args += ['-DZSTD_HEAPMODE=0', + '-D_CRT_SECURE_NO_WARNINGS'] + endif +endif + +mingw_ansi_stdio_flags = [] +if host_machine_os == os_windows and cc_id == compiler_gcc + mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ] +endif +libzstd_c_args += mingw_ansi_stdio_flags + +libzstd_debug_cflags = [] +if enable_debug and meson_buildtype == 'debug' + if cc_id == compiler_gcc or cc_id == compiler_clang + libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum', + '-Wdeclaration-after-statement', '-Wstrict-prototypes', + '-Wundef', '-Wpointer-arith', '-Wformat-security', '-Wvla', + '-Wformat=2', '-Winit-self', '-Wfloat-equal', '-Wwrite-strings', + '-Wredundant-decls', '-Wmissing-prototypes', '-Wc++-compat'] + endif +endif +libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags) + libzstd = library('zstd', - libzstd_sources, - include_directories: libzstd_includes, - dependencies: libzstd_deps, - install: true, - soversion: zstd_version) + libzstd_sources, + include_directories: libzstd_includes, + c_args: libzstd_c_args, + dependencies: libzstd_deps, + install: true, + soversion: zstd_libversion) + +libzstd_dep = declare_dependency(link_with: libzstd, + include_directories: libzstd_includes) pkgconfig.generate(name: 'libzstd', - filebase: 'libzstd', - libraries: [libzstd], - description: 'fast lossless compression algorithm library', - version: zstd_version, - url: 'http://www.zstd.net/') + filebase: 'libzstd', + libraries: [libzstd], + description: 'fast lossless compression algorithm library', + version: zstd_libversion, + url: 'http://www.zstd.net/') -install_headers(join_paths(library_dir, 'zstd.h'), - join_paths(library_deprecated_dir, 'zbuff.h'), - join_paths(library_dictbuilder_dir, 'zdict.h'), - join_paths(library_common_dir, 'zstd_errors.h')) +install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'), + join_paths(zstd_rootdir, 'lib/deprecated/zbuff.h'), + join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'), + join_paths(zstd_rootdir, 'lib/common/zstd_errors.h')) diff --git a/build/meson/meson.build b/build/meson/meson.build index 750e841cf..f4c6f32d5 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -10,7 +10,7 @@ project('zstd', ['c', 'cpp'], - license: 'BSD', + license: ['BSD', 'GPLv2'], default_options : ['c_std=c99', 'cpp_std=c++11', 'buildtype=release'], @@ -22,36 +22,65 @@ cc = meson.get_compiler('c') cxx = meson.get_compiler('cpp') pkgconfig = import('pkgconfig') python3 = import('python').find_installation() +windows_mod = import('windows') + +host_machine_os = host_machine.system() +os_windows = 'windows' +os_linux = 'linux' +os_darwin = 'darwin' +os_freebsd = 'freebsd' +os_sun = 'sunos' + +cc_id = cc.get_id() +compiler_gcc = 'gcc' +compiler_clang = 'clang' +compiler_msvc = 'msvc' zstd_version = meson.project_version() +zstd_libversion = '' # ============================================================================= # Project directories # ============================================================================= -zstd_prefix = get_option('prefix') -zstd_bindir = join_paths(zstd_prefix, get_option('bindir')) -zstd_datadir = join_paths(zstd_prefix, get_option('datadir')) -zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name()) -zstd_mandir = join_paths(zstd_prefix, get_option('mandir')) +zstd_rootdir = '../..' -zstd_source_dir = join_paths('..', '..') -library_dir = join_paths(zstd_source_dir, 'lib') -contrib_meson_dir = join_paths(zstd_source_dir, 'contrib', 'meson') +# ============================================================================= +# Installing directories +# ============================================================================= + +if host_machine_os == os_windows + zstd_prefix = '.' + zstd_bindir = 'bin' + zstd_datadir = 'share' + zstd_mandir = join_paths(zstd_datadir, 'man') +else + zstd_prefix = get_option('prefix') + zstd_bindir = join_paths(zstd_prefix, get_option('bindir')) + zstd_datadir = join_paths(zstd_prefix, get_option('datadir')) + zstd_mandir = join_paths(zstd_prefix, get_option('mandir')) +endif + +zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name()) # ============================================================================= # Project options # ============================================================================= enable_debug = get_option('debug') +default_library_type = get_option('default_library') +meson_buildtype = get_option('buildtype') with_legacy_support = get_option('with-legacy-support') with_programs = get_option('with-programs') with_tests = get_option('with-tests') with_contrib = get_option('with-contrib') +with_debug_level = get_option('with-debug-level') enable_multithread = get_option('enable-multithread') +enable_static_runtime = get_option('enable-static-runtime') enable_zlib = get_option('enable-zlib') enable_lzma = get_option('enable-lzma') enable_lz4 = get_option('enable-lz4') +enable_backtrace = get_option('enable-backtrace') # ============================================================================= # Helper scripts for Meson @@ -63,7 +92,7 @@ GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py') # Getting project version from zstd.h # ============================================================================= -zstd_h_file = join_paths(meson.current_source_dir(), library_dir, 'zstd.h') +zstd_h_file = join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h') r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file) if r.returncode() == 0 output = r.stdout().strip() @@ -73,31 +102,42 @@ if r.returncode() == 0 endif endif +if host_machine_os != os_windows + zstd_libversion = zstd_version +endif + # ============================================================================= # Dependencies # ============================================================================= -libm_dep = cc.find_library('m', required: true) -thread_dep = dependency('threads', required: false) +libm_dep = cc.find_library('m', required: with_tests) +thread_dep = dependency('threads', required: enable_multithread) # Arguments in dependency should be equivalent to those passed to pkg-config -zlib_dep = dependency('zlib', required: false) -lzma_dep = dependency('liblzma', required: false) -lz4_dep = dependency('liblz4', required: false) +zlib_dep = dependency('zlib', required: enable_zlib) +lzma_dep = dependency('liblzma', required: enable_lzma) +lz4_dep = dependency('liblz4', required: enable_lz4) # ============================================================================= # Compiler flags # ============================================================================= -if cxx.get_id() == 'gcc' or cxx.get_id() == 'clang' - common_flags = [ '-DXXH_NAMESPACE=ZSTD_' ] +add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: [ 'c' ]) + +if [compiler_gcc, compiler_clang].contains(cc_id) common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ] - cc_compilation_flags = cc.get_supported_arguments(common_warning_flags) - cc_compilation_flags += cc.get_supported_arguments(['-Wstrict-prototypes']) - cc_compilation_flags += common_flags - cxx_compilation_flags = cxx.get_supported_arguments(common_warning_flags) - cxx_compilation_flags += common_flags - add_project_arguments(cc_compilation_flags, language : 'c') - add_project_arguments(cxx_compilation_flags, language : 'cpp') + cc_compile_flags = cc.get_supported_arguments(common_warning_flags + [ '-Wstrict-prototypes' ]) + cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags) + add_project_arguments(cc_compile_flags, language : 'c') + add_project_arguments(cxx_compile_flags, language : 'cpp') +elif cc_id == compiler_msvc + msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ] + if enable_multithread + msvc_compile_flags += [ '/MP' ] + endif + if enable_static_runtime + msvc_compile_flags += [ '/MT' ] + endif + add_project_arguments(msvc_compile_flags, language: ['c', 'cpp']) endif # ============================================================================= diff --git a/build/meson/meson_options.txt b/build/meson/meson_options.txt index dca6df809..d4bac3c48 100644 --- a/build/meson/meson_options.txt +++ b/build/meson/meson_options.txt @@ -8,21 +8,25 @@ # in the COPYING file in the root directory of this source tree). # ############################################################################# -option('enable-multithread', type: 'boolean', value: true, - description: 'Enable multi-threading when pthread is detected') -option('with-legacy-support', type: 'string', value: '5', - description: 'Support any legacy format: true or false, or 7 to 1 for v0.7+ to v0.1+') +option('with-legacy-support', type: 'integer', min: 0, max: 7, value: '5', + description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+') option('with-programs', type: 'boolean', value: true, - description: 'Enable programs build') + description: 'Enable programs build') option('with-contrib', type: 'boolean', value: false, - description: 'Enable contrib build') + description: 'Enable contrib build') option('with-tests', type: 'boolean', value: false, - description: 'Enable tests build') + description: 'Enable tests build') +option('with-debug-level', type: 'integer', min: 0, max: 9, value: 1, # Since 0.45.0 + description: 'Enable run-time debug. See lib/common/debug.h') +option('enable-multithread', type: 'boolean', value: true, + description: 'Enable multi-threading when pthread is detected') option('enable-static-runtime', type: 'boolean', value: false, - description: 'Link to static run-time libraries on MSVC') + description: 'Link to static run-time libraries on MSVC') option('enable-zlib', type: 'boolean', value: false, - description: 'Enable zlib support') + description: 'Enable zlib support') option('enable-lzma', type: 'boolean', value: false, - description: 'Enable lzma support') + description: 'Enable lzma support') option('enable-lz4', type: 'boolean', value: false, - description: 'Enable lz4 support') + description: 'Enable lz4 support') +option('enable-backtrace', type: 'boolean', value: false, + description: 'Display a stack backtrace when execution generates a runtime exception. Only in debug build mode.') diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build index c96b6d458..8102eda15 100644 --- a/build/meson/programs/meson.build +++ b/build/meson/programs/meson.build @@ -8,74 +8,86 @@ # in the COPYING file in the root directory of this source tree). # ############################################################################# -zstd_source_dir = join_paths('..', '..', '..') -programs_dir = join_paths(zstd_source_dir, 'programs') +zstd_rootdir = '../../..' -zstdcli_c_file = join_paths(programs_dir, 'zstdcli.c') -util_c_file = join_paths(programs_dir, 'util.c') -fileio_c_file = join_paths(programs_dir, 'fileio.c') -zstd_programs_sources = [zstdcli_c_file, - util_c_file, - fileio_c_file, - join_paths(programs_dir, 'benchfn.c'), - join_paths(programs_dir, 'benchzstd.c'), - join_paths(programs_dir, 'datagen.c'), - join_paths(programs_dir, 'dibio.c')] +zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'), + join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'programs/fileio.c'), + join_paths(zstd_rootdir, 'programs/benchfn.c'), + join_paths(zstd_rootdir, 'programs/benchzstd.c'), + join_paths(zstd_rootdir, 'programs/datagen.c'), + join_paths(zstd_rootdir, 'programs/dibio.c')] -zstd_c_args = [] +zstd_c_args = libzstd_debug_cflags if enable_multithread - zstd_c_args += [ '-DZSTD_MULTITHREAD' ] + zstd_c_args += [ '-DZSTD_MULTITHREAD' ] endif -zstd_deps = [] +zstd_deps = [ libzstd_dep ] if enable_zlib and zlib_dep.found() - zstd_deps += [ zlib_dep ] - zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ] + zstd_deps += [ zlib_dep ] + zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ] endif if enable_lzma and lzma_dep.found() - zstd_deps += [ lzma_dep ] - zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ] + zstd_deps += [ lzma_dep ] + zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ] endif if enable_lz4 and lz4_dep.found() - zstd_deps += [ lz4_dep ] - zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ] + zstd_deps += [ lz4_dep ] + zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ] +endif + +export_dynamic_on_windows = false +# explicit backtrace enable/disable for Linux & Darwin +if not enable_backtrace + zstd_c_args += '-DBACKTRACE_ENABLE=0' +elif enable_debug and host_machine_os == os_windows # MinGW target + zstd_c_args += '-DBACKTRACE_ENABLE=1' + export_dynamic_on_windows = true +endif + +if cc_id == compiler_msvc + if default_library_type != 'static' + zstd_programs_sources += [windows_mod.compile_resources( + join_paths(zstd_rootdir, 'build/VS2010/zstd/zstd.rc'))] + endif endif zstd = executable('zstd', - zstd_programs_sources, - c_args: zstd_c_args, - include_directories: libzstd_includes, - link_with: libzstd, - dependencies: zstd_deps, - install: true) + zstd_programs_sources, + c_args: zstd_c_args, + dependencies: zstd_deps, + export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0 + install: true) -zstd_frugal_sources = [join_paths(programs_dir, 'zstdcli.c'), - util_c_file, - fileio_c_file] +zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'), + join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'programs/fileio.c')] +# Minimal target, with only zstd compression and decompression. +# No bench. No legacy. executable('zstd-frugal', - zstd_frugal_sources, - include_directories: libzstd_includes, - link_with: libzstd, - c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ], - install: true) + zstd_frugal_sources, + dependencies: libzstd_dep, + c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ], + install: true) -install_data(join_paths(programs_dir, 'zstdgrep'), - join_paths(programs_dir, 'zstdless'), - install_dir: zstd_bindir) +install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'), + join_paths(zstd_rootdir, 'programs/zstdless'), + install_dir: zstd_bindir) # ============================================================================= # Program symlinks # ============================================================================= -InstallSymlink_py = join_paths('..', 'InstallSymlink.py') +InstallSymlink_py = '../InstallSymlink.py' meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdcat', zstd_bindir) meson.add_install_script(InstallSymlink_py, 'zstd', 'unzstd', zstd_bindir) if enable_multithread - meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdmt', zstd_bindir) + meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdmt', zstd_bindir) endif # ============================================================================= @@ -84,9 +96,9 @@ endif zstd_man1_dir = join_paths(zstd_mandir, 'man1') -install_man(join_paths(programs_dir, 'zstd.1'), - join_paths(programs_dir, 'zstdgrep.1'), - join_paths(programs_dir, 'zstdless.1')) +install_man(join_paths(zstd_rootdir, 'programs/zstd.1'), + join_paths(zstd_rootdir, 'programs/zstdgrep.1'), + join_paths(zstd_rootdir, 'programs/zstdless.1')) # Meson automatically compresses manpages to gz format # WARNING: This may fail on Windows. Test NEEDED. @@ -94,5 +106,5 @@ meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdcat.1.gz', zstd_ma meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'unzstd.1.gz', zstd_man1_dir) if enable_multithread - meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdmt.1.gz', zstd_man1_dir) + meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdmt.1.gz', zstd_man1_dir) endif From c55d65b35db402b4f3838994226e7c52be7626a3 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 30 Nov 2018 16:15:33 +0700 Subject: [PATCH 38/62] Add clang cflags --- build/meson/meson.build | 83 +++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/build/meson/meson.build b/build/meson/meson.build index f4c6f32d5..f181fdc32 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -9,14 +9,14 @@ # ############################################################################# project('zstd', - ['c', 'cpp'], - license: ['BSD', 'GPLv2'], - default_options : ['c_std=c99', - 'cpp_std=c++11', - 'buildtype=release'], - version: '1.3.8', - # for install_man - meson_version: '>=0.47.0') + ['c', 'cpp'], + license: ['BSD', 'GPLv2'], + default_options : ['c_std=c99', + 'cpp_std=c++11', + 'buildtype=release'], + version: '1.3.8', + # for install_man + meson_version: '>=0.47.0') cc = meson.get_compiler('c') cxx = meson.get_compiler('cpp') @@ -46,19 +46,19 @@ zstd_libversion = '' zstd_rootdir = '../..' # ============================================================================= -# Installing directories +# Installation directories # ============================================================================= if host_machine_os == os_windows - zstd_prefix = '.' - zstd_bindir = 'bin' - zstd_datadir = 'share' - zstd_mandir = join_paths(zstd_datadir, 'man') + zstd_prefix = '.' + zstd_bindir = 'bin' + zstd_datadir = 'share' + zstd_mandir = join_paths(zstd_datadir, 'man') else - zstd_prefix = get_option('prefix') - zstd_bindir = join_paths(zstd_prefix, get_option('bindir')) - zstd_datadir = join_paths(zstd_prefix, get_option('datadir')) - zstd_mandir = join_paths(zstd_prefix, get_option('mandir')) + zstd_prefix = get_option('prefix') + zstd_bindir = join_paths(zstd_prefix, get_option('bindir')) + zstd_datadir = join_paths(zstd_prefix, get_option('datadir')) + zstd_mandir = join_paths(zstd_prefix, get_option('mandir')) endif zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name()) @@ -95,15 +95,15 @@ GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py') zstd_h_file = join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h') r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file) if r.returncode() == 0 - output = r.stdout().strip() - if output.version_compare('>@0@'.format(zstd_version)) - zstd_version = output - message('Project version is now: @0@'.format(zstd_version)) - endif + output = r.stdout().strip() + if output.version_compare('>@0@'.format(zstd_version)) + zstd_version = output + message('Project version is now: @0@'.format(zstd_version)) + endif endif if host_machine_os != os_windows - zstd_libversion = zstd_version + zstd_libversion = zstd_version endif # ============================================================================= @@ -121,23 +121,26 @@ lz4_dep = dependency('liblz4', required: enable_lz4) # Compiler flags # ============================================================================= -add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: [ 'c' ]) +add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c']) if [compiler_gcc, compiler_clang].contains(cc_id) - common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ] - cc_compile_flags = cc.get_supported_arguments(common_warning_flags + [ '-Wstrict-prototypes' ]) - cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags) - add_project_arguments(cc_compile_flags, language : 'c') - add_project_arguments(cxx_compile_flags, language : 'cpp') + common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ] + if cc_id == compiler_clang + common_warning_flags += ['-Werror', '-Wconversion', '-Wno-sign-conversion', '-Wdocumentation'] + endif + cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes']) + cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags) + add_project_arguments(cc_compile_flags, language : 'c') + add_project_arguments(cxx_compile_flags, language : 'cpp') elif cc_id == compiler_msvc - msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ] - if enable_multithread - msvc_compile_flags += [ '/MP' ] - endif - if enable_static_runtime - msvc_compile_flags += [ '/MT' ] - endif - add_project_arguments(msvc_compile_flags, language: ['c', 'cpp']) + msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ] + if enable_multithread + msvc_compile_flags += '/MP' + endif + if enable_static_runtime + msvc_compile_flags += '/MT' + endif + add_project_arguments(msvc_compile_flags, language: ['c', 'cpp']) endif # ============================================================================= @@ -147,13 +150,13 @@ endif subdir('lib') if with_programs - subdir('programs') + subdir('programs') endif if with_tests - subdir('tests') + subdir('tests') endif if with_contrib - subdir('contrib') + subdir('contrib') endif From 39f49ac39f8f0bd357049a066bcf377749d5d2a5 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 30 Nov 2018 17:23:23 +0700 Subject: [PATCH 39/62] Add almost all test cases in tests/Makefile --- build/meson/tests/meson.build | 218 ++++++++++++++++++++++++------ build/meson/tests/valgrindTest.py | 90 ++++++++++++ 2 files changed, 263 insertions(+), 45 deletions(-) create mode 100644 build/meson/tests/valgrindTest.py diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build index c64ae7a7a..4e127dec3 100644 --- a/build/meson/tests/meson.build +++ b/build/meson/tests/meson.build @@ -8,58 +8,186 @@ # in the COPYING file in the root directory of this source tree). # ############################################################################# -zstd_source_dir = join_paths('..', '..', '..') -programs_dir = join_paths(zstd_source_dir, 'programs') -tests_dir = join_paths(zstd_source_dir, 'tests') +zstd_rootdir = '../../..' -datagen_c_file = join_paths(programs_dir, 'datagen.c') -util_c_file = join_paths(programs_dir, 'util.c') -benchfn_c_file = join_paths(programs_dir, 'benchfn.c') +tests_supported_oses = [os_linux, 'gnu/kfreebsd', os_darwin, 'gnu', 'openbsd', + os_freebsd, 'netbsd', 'dragonfly', os_sun] -datagen_sources = [datagen_c_file, - join_paths(tests_dir, 'datagencli.c')] -test_includes = libzstd_includes + [include_directories(programs_dir)] +# ============================================================================= +# Test flags +# ============================================================================= +FUZZER_FLAGS = ['--no-big-tests'] +FUZZERTEST = '-T200s' +ZSTREAM_TESTTIME = '-T90s' +DECODECORPUS_TESTTIME = '-T30' +ZSTDRTTEST = ['--test-large-data'] + +# ============================================================================= +# Executables +# ============================================================================= + +test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ] + +datagen_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'), + join_paths(zstd_rootdir, 'tests/datagencli.c')] datagen = executable('datagen', - datagen_sources, - include_directories: test_includes, - link_with: libzstd, - install: false) -test('datagen', datagen) + datagen_sources, + c_args: [ '-DNDEBUG' ], + include_directories: test_includes, + dependencies: libzstd_dep, + install: false) -fullbench_sources = [datagen_c_file, - util_c_file, - benchfn_c_file, - join_paths(programs_dir, 'benchzstd.c'), - join_paths(tests_dir, 'fullbench.c')] +fullbench_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'), + join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'programs/benchfn.c'), + join_paths(zstd_rootdir, 'programs/benchzstd.c'), + join_paths(zstd_rootdir, 'tests/fullbench.c')] fullbench = executable('fullbench', - fullbench_sources, - include_directories: test_includes, - link_with: libzstd, - install: false) -test('fullbench', fullbench) + fullbench_sources, + include_directories: test_includes, + dependencies: libzstd_dep, + install: false) -fuzzer_sources = [datagen_c_file, - util_c_file, - join_paths(tests_dir, 'fuzzer.c')] +fuzzer_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'), + join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'tests/fuzzer.c')] fuzzer = executable('fuzzer', - fuzzer_sources, - include_directories: test_includes, - link_with: libzstd, - install: false) -test('fuzzer', fuzzer) + fuzzer_sources, + include_directories: test_includes, + dependencies: libzstd_dep, + install: false) -paramgrill_sources = [benchfn_c_file, - join_paths(programs_dir, 'benchzstd.c'), - datagen_c_file, - util_c_file, - join_paths(tests_dir, 'paramgrill.c')] -if host_machine.system() != 'windows' - paramgrill = executable('paramgrill', - paramgrill_sources, - include_directories: test_includes, - link_with: libzstd, - dependencies: libm_dep, - install: false) - test('paramgrill', paramgrill) +zbufftest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'), + join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'tests/zbufftest.c')] +zbufftest = executable('zbufftest', + zbufftest_sources, + c_args: ['-Wno-deprecated-declarations'], + include_directories: test_includes, + dependencies: libzstd_dep, + install: false) + +zstreamtest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'), + join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'tests/seqgen.c'), + join_paths(zstd_rootdir, 'tests/zstreamtest.c')] +zstreamtest = executable('zstreamtest', + zstreamtest_sources, + include_directories: test_includes, + dependencies: libzstd_dep, + install: false) + +paramgrill_sources = [join_paths(zstd_rootdir, 'programs/benchfn.c'), + join_paths(zstd_rootdir, 'programs/benchzstd.c'), + join_paths(zstd_rootdir, 'programs/datagen.c'), + join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'tests/paramgrill.c')] +paramgrill = executable('paramgrill', + paramgrill_sources, + include_directories: test_includes, + dependencies: [ libzstd_dep, libm_dep ], + install: false) + +roundTripCrash_sources = [join_paths(zstd_rootdir, 'tests/roundTripCrash.c')] +roundTripCrash = executable('roundTripCrash', + roundTripCrash_sources, + dependencies: [ libzstd_dep ], + install: false) + +longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')] +longmatch = executable('longmatch', + longmatch_sources, + dependencies: [ libzstd_dep ], + install: false) + +invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')] +invalidDictionaries = executable('invalidDictionaries', + invalidDictionaries_sources, + dependencies: [ libzstd_dep ], + install: false) + +legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')] +legacy = executable('legacy', + legacy_sources, + c_args: '-DZSTD_LEGACY_SUPPORT=4', + dependencies: [ libzstd_dep ], + install: false) + +decodecorpus_sources = [join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'tests/decodecorpus.c')] +decodecorpus = executable('decodecorpus', + decodecorpus_sources, + include_directories: test_includes, + dependencies: [ libzstd_dep, libm_dep ], + install: false) + +symbols_sources = [join_paths(zstd_rootdir, 'tests/symbols.c')] +symbols = executable('symbols', + symbols_sources, + include_directories: test_includes, + c_args: host_machine_os == os_windows ? '-DZSTD_DLL_IMPORT=1' : [], + dependencies: [ libzstd_dep ], + install: false) + +poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'), + join_paths(zstd_rootdir, 'tests/poolTests.c'), + join_paths(zstd_rootdir, 'lib/common/pool.c'), + join_paths(zstd_rootdir, 'lib/common/threading.c'), + join_paths(zstd_rootdir, 'lib/common/zstd_common.c'), + join_paths(zstd_rootdir, 'lib/common/error_private.c')] +poolTests = executable('poolTests', + poolTests_sources, + include_directories: test_includes, + dependencies: [ libzstd_dep ], + install: false) + +checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')] +checkTag = executable('checkTag', + checkTag_sources, + dependencies: [ libzstd_dep ], + install: false) + +# ============================================================================= +# Tests (Use "meson test --list" to list all tests) +# ============================================================================= + +if tests_supported_oses.contains(host_machine_os) + valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true) + valgrindTest_py = files('valgrindTest.py') + test('valgrindTest', + valgrindTest_py, + args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench], + depends: [zstd, datagen, fuzzer, fullbench], + timeout: 600) # Timeout should work on HDD drive endif + +if host_machine_os != os_windows + playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true) + test('test-zstd', + playTests_sh, + args: ZSTDRTTEST, + env: ['ZSTD=' + zstd.full_path()], + depends: [datagen], + timeout: 600) # Timeout should work on HDD drive +endif + +test('test-fullbench-1', fullbench, args: ['-i1'], + depends: [fullbench, datagen]) +test('test-fullbench-2', fullbench, args: ['-i1', '-P0'], + depends: [fullbench, datagen]) + +if enable_zlib + test('test-fuzzer', fuzzer, args: ['-v', FUZZERTEST] + FUZZER_FLAGS) +endif + +test('test-zbuff', zbufftest, args: [ZSTREAM_TESTTIME]) +test('test-zstream-1', zstreamtest, args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS) +test('test-zstream-2', zstreamtest, args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS) +test('test-zstream-3', zstreamtest, args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS) +test('test-longmatch', longmatch) +test('test-invalidDictionaries', invalidDictionaries) +test('test-symbols', symbols) +test('test-legacy', legacy) +test('test-decodecorpus', decodecorpus, args: ['-t', DECODECORPUS_TESTTIME]) +test('test-poolTests', poolTests) diff --git a/build/meson/tests/valgrindTest.py b/build/meson/tests/valgrindTest.py new file mode 100644 index 000000000..218f7458b --- /dev/null +++ b/build/meson/tests/valgrindTest.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# ############################################################################# +# Copyright (c) 2018-present lzutao +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ############################################################################# +import os +import subprocess +import tempfile + + +def valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench): + VALGRIND_ARGS = [valgrind, '--leak-check=full', '--show-leak-kinds=all', '--error-exitcode=1'] + + print('\n ---- valgrind tests : memory analyzer ----') + + subprocess.check_call([*VALGRIND_ARGS, datagen, '-g50M'], stdout=subprocess.DEVNULL) + + if subprocess.call([*VALGRIND_ARGS, zstd], + stdout=subprocess.DEVNULL) == 0: + raise subprocess.CalledProcessError('zstd without argument should have failed') + + with subprocess.Popen([datagen, '-g80'], stdout=subprocess.PIPE) as p1, \ + subprocess.Popen([*VALGRIND_ARGS, zstd, '-', '-c'], + stdin=p1.stdout, + stdout=subprocess.DEVNULL) as p2: + p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. + p2.communicate() + if p2.returncode != 0: + raise subprocess.CalledProcessError() + + with subprocess.Popen([datagen, '-g16KB'], stdout=subprocess.PIPE) as p1, \ + subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'], + stdin=p1.stdout, + stdout=subprocess.DEVNULL) as p2: + p1.stdout.close() + p2.communicate() + if p2.returncode != 0: + raise subprocess.CalledProcessError() + + with tempfile.NamedTemporaryFile() as tmp_fd: + with subprocess.Popen([datagen, '-g2930KB'], stdout=subprocess.PIPE) as p1, \ + subprocess.Popen([*VALGRIND_ARGS, zstd, '-5', '-vf', '-', '-o', tmp_fd.name], + stdin=p1.stdout) as p2: + p1.stdout.close() + p2.communicate() + if p2.returncode != 0: + raise subprocess.CalledProcessError() + + subprocess.check_call([*VALGRIND_ARGS, zstd, '-vdf', tmp_fd.name, '-c'], + stdout=subprocess.DEVNULL) + + with subprocess.Popen([datagen, '-g64MB'], stdout=subprocess.PIPE) as p1, \ + subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'], + stdin=p1.stdout, + stdout=subprocess.DEVNULL) as p2: + p1.stdout.close() + p2.communicate() + if p2.returncode != 0: + raise subprocess.CalledProcessError() + + subprocess.check_call([*VALGRIND_ARGS, fuzzer, '-T1mn', '-t1']) + subprocess.check_call([*VALGRIND_ARGS, fullbench, '-i1']) + + +def main(): + import argparse + parser = argparse.ArgumentParser(description='Valgrind tests : memory analyzer') + parser.add_argument('valgrind', help='valgrind path') + parser.add_argument('zstd', help='zstd path') + parser.add_argument('datagen', help='datagen path') + parser.add_argument('fuzzer', help='fuzzer path') + parser.add_argument('fullbench', help='fullbench path') + + args = parser.parse_args() + + valgrind = args.valgrind + zstd = args.zstd + datagen = args.datagen + fuzzer = args.fuzzer + fullbench = args.fullbench + + valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench) + + +if __name__ == '__main__': + main() From 24bc513ea16e1e6e53e42d351dfec4035755fd8f Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 30 Nov 2018 21:05:03 +0700 Subject: [PATCH 40/62] meson: Change build options' name --- build/meson/lib/meson.build | 13 ++++---- build/meson/meson.build | 54 ++++++++++++++++++-------------- build/meson/meson_options.txt | 36 +++++++++++---------- build/meson/programs/meson.build | 16 +++++----- build/meson/tests/meson.build | 4 +-- 5 files changed, 67 insertions(+), 56 deletions(-) diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 5bfdbb074..c39a65530 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -47,25 +47,25 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')] # Explicit define legacy support -add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(with_legacy_support), +add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level), language: 'c') -if with_legacy_support == 0 +if legacy_level == 0 message('Legacy support: DISABLED') else # See ZSTD_LEGACY_SUPPORT of lib/README.md - message('Enable legacy support back to version 0.@0@'.format(with_legacy_support)) + message('Enable legacy support back to version 0.@0@'.format(legacy_level)) libzstd_includes += [ include_directories(join_paths(zstd_rootdir, 'lib/legacy')) ] foreach i : [1, 2, 3, 4, 5, 6, 7] - if with_legacy_support <= i + if legacy_level <= i libzstd_sources += join_paths(zstd_rootdir, 'lib/legacy/zstd_v0@0@.c'.format(i)) endif endforeach endif libzstd_deps = [] -if enable_multithread +if use_multi_thread message('Enable multi-threading support') add_project_arguments('-DZSTD_MULTITHREAD', language: 'c') libzstd_deps = [ thread_dep ] @@ -93,7 +93,8 @@ endif libzstd_c_args += mingw_ansi_stdio_flags libzstd_debug_cflags = [] -if enable_debug and meson_buildtype == 'debug' +if use_debug + libzstd_c_args += '-DDEBUGLEVEL=@0@'.format(debug_level) if cc_id == compiler_gcc or cc_id == compiler_clang libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum', '-Wdeclaration-after-statement', '-Wstrict-prototypes', diff --git a/build/meson/meson.build b/build/meson/meson.build index f181fdc32..c36c9425a 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -15,7 +15,6 @@ project('zstd', 'cpp_std=c++11', 'buildtype=release'], version: '1.3.8', - # for install_man meson_version: '>=0.47.0') cc = meson.get_compiler('c') @@ -67,20 +66,23 @@ zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name()) # Project options # ============================================================================= -enable_debug = get_option('debug') -default_library_type = get_option('default_library') -meson_buildtype = get_option('buildtype') -with_legacy_support = get_option('with-legacy-support') -with_programs = get_option('with-programs') -with_tests = get_option('with-tests') -with_contrib = get_option('with-contrib') -with_debug_level = get_option('with-debug-level') -enable_multithread = get_option('enable-multithread') -enable_static_runtime = get_option('enable-static-runtime') -enable_zlib = get_option('enable-zlib') -enable_lzma = get_option('enable-lzma') -enable_lz4 = get_option('enable-lz4') -enable_backtrace = get_option('enable-backtrace') +# Built-in options +use_debug = get_option('debug') + +# Custom options +debug_level = get_option('debug_level') +legacy_level = get_option('legacy_level') +use_backtrace = get_option('backtrace') +use_static_runtime = get_option('static_runtime') + +build_programs = get_option('build_programs') +build_contrib = get_option('build_contrib') +build_tests = get_option('build_tests') + +feature_multi_thread = get_option('multi_thread') +feature_zlib = get_option('zlib') +feature_lzma = get_option('lzma') +feature_lz4 = get_option('lz4') # ============================================================================= # Helper scripts for Meson @@ -110,12 +112,16 @@ endif # Dependencies # ============================================================================= -libm_dep = cc.find_library('m', required: with_tests) -thread_dep = dependency('threads', required: enable_multithread) +libm_dep = cc.find_library('m', required: build_tests) +thread_dep = dependency('threads', required: feature_multi_thread) +use_multi_thread = thread_dep.found() # Arguments in dependency should be equivalent to those passed to pkg-config -zlib_dep = dependency('zlib', required: enable_zlib) -lzma_dep = dependency('liblzma', required: enable_lzma) -lz4_dep = dependency('liblz4', required: enable_lz4) +zlib_dep = dependency('zlib', required: feature_zlib) +use_zlib = zlib_dep.found() +lzma_dep = dependency('liblzma', required: feature_lzma) +use_lzma = lzma_dep.found() +lz4_dep = dependency('liblz4', required: feature_lz4) +use_lz4 = lz4_dep.found() # ============================================================================= # Compiler flags @@ -134,7 +140,7 @@ if [compiler_gcc, compiler_clang].contains(cc_id) add_project_arguments(cxx_compile_flags, language : 'cpp') elif cc_id == compiler_msvc msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ] - if enable_multithread + if use_multi_thread msvc_compile_flags += '/MP' endif if enable_static_runtime @@ -149,14 +155,14 @@ endif subdir('lib') -if with_programs +if build_programs subdir('programs') endif -if with_tests +if build_tests subdir('tests') endif -if with_contrib +if build_contrib subdir('contrib') endif diff --git a/build/meson/meson_options.txt b/build/meson/meson_options.txt index d4bac3c48..349d915c7 100644 --- a/build/meson/meson_options.txt +++ b/build/meson/meson_options.txt @@ -8,25 +8,29 @@ # in the COPYING file in the root directory of this source tree). # ############################################################################# -option('with-legacy-support', type: 'integer', min: 0, max: 7, value: '5', +# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting + +option('legacy_level', type: 'integer', min: 0, max: 7, value: '5', description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+') -option('with-programs', type: 'boolean', value: true, - description: 'Enable programs build') -option('with-contrib', type: 'boolean', value: false, - description: 'Enable contrib build') -option('with-tests', type: 'boolean', value: false, - description: 'Enable tests build') -option('with-debug-level', type: 'integer', min: 0, max: 9, value: 1, # Since 0.45.0 +option('debug_level', type: 'integer', min: 0, max: 9, value: 1, description: 'Enable run-time debug. See lib/common/debug.h') -option('enable-multithread', type: 'boolean', value: true, - description: 'Enable multi-threading when pthread is detected') -option('enable-static-runtime', type: 'boolean', value: false, +option('backtrace', type: 'boolean', value: false, + description: 'Display a stack backtrace when execution generates a runtime exception') +option('static_runtime', type: 'boolean', value: false, description: 'Link to static run-time libraries on MSVC') -option('enable-zlib', type: 'boolean', value: false, + +option('build_programs', type: 'boolean', value: true, + description: 'Enable programs build') +option('build_tests', type: 'boolean', value: false, + description: 'Enable tests build') +option('build_contrib', type: 'boolean', value: false, + description: 'Enable contrib build') + +option('multi_thread', type: 'feature', value: 'enabled', + description: 'Enable multi-threading when pthread is detected') +option('zlib', type: 'feature', value: 'auto', description: 'Enable zlib support') -option('enable-lzma', type: 'boolean', value: false, +option('lzma', type: 'feature', value: 'auto', description: 'Enable lzma support') -option('enable-lz4', type: 'boolean', value: false, +option('lz4', type: 'feature', value: 'auto', description: 'Enable lz4 support') -option('enable-backtrace', type: 'boolean', value: false, - description: 'Display a stack backtrace when execution generates a runtime exception. Only in debug build mode.') diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build index 8102eda15..f538aa556 100644 --- a/build/meson/programs/meson.build +++ b/build/meson/programs/meson.build @@ -19,31 +19,31 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'), join_paths(zstd_rootdir, 'programs/dibio.c')] zstd_c_args = libzstd_debug_cflags -if enable_multithread +if use_multi_thread zstd_c_args += [ '-DZSTD_MULTITHREAD' ] endif zstd_deps = [ libzstd_dep ] -if enable_zlib and zlib_dep.found() +if use_zlib zstd_deps += [ zlib_dep ] zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ] endif -if enable_lzma and lzma_dep.found() +if use_lzma zstd_deps += [ lzma_dep ] zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ] endif -if enable_lz4 and lz4_dep.found() +if use_lz4 zstd_deps += [ lz4_dep ] zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ] endif export_dynamic_on_windows = false # explicit backtrace enable/disable for Linux & Darwin -if not enable_backtrace +if not use_backtrace zstd_c_args += '-DBACKTRACE_ENABLE=0' -elif enable_debug and host_machine_os == os_windows # MinGW target +elif use_debug and host_machine_os == os_windows # MinGW target zstd_c_args += '-DBACKTRACE_ENABLE=1' export_dynamic_on_windows = true endif @@ -86,7 +86,7 @@ InstallSymlink_py = '../InstallSymlink.py' meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdcat', zstd_bindir) meson.add_install_script(InstallSymlink_py, 'zstd', 'unzstd', zstd_bindir) -if enable_multithread +if use_multi_thread meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdmt', zstd_bindir) endif @@ -105,6 +105,6 @@ install_man(join_paths(zstd_rootdir, 'programs/zstd.1'), meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdcat.1.gz', zstd_man1_dir) meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'unzstd.1.gz', zstd_man1_dir) -if enable_multithread +if use_multi_thread meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdmt.1.gz', zstd_man1_dir) endif diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build index 4e127dec3..c8772e761 100644 --- a/build/meson/tests/meson.build +++ b/build/meson/tests/meson.build @@ -139,7 +139,7 @@ poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'), poolTests = executable('poolTests', poolTests_sources, include_directories: test_includes, - dependencies: [ libzstd_dep ], + dependencies: [ libzstd_dep, thread_dep ], install: false) checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')] @@ -177,7 +177,7 @@ test('test-fullbench-1', fullbench, args: ['-i1'], test('test-fullbench-2', fullbench, args: ['-i1', '-P0'], depends: [fullbench, datagen]) -if enable_zlib +if use_zlib test('test-fuzzer', fuzzer, args: ['-v', FUZZERTEST] + FUZZER_FLAGS) endif From ff1bca3fbd4c427f7bd4671eaf40cd9778fa281e Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 30 Nov 2018 21:10:06 +0700 Subject: [PATCH 41/62] ci: Use new meson auto_features option * Install lz4 dep from Ubuntu repo * Use curl instead of wget * Use xenial for meson and make meson build first * Unset Travis preset CC and CXX value for meson to use ccache * Be more verbose with "set -x" --- .travis.yml | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a0170e04..1ed148d0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,8 @@ matrix: - if: tag =~ ^v[0-9]\.[0-9] env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH' - - env: BUILD_SYSTEM='meson' + - dist: xenial + env: BUILD_SYSTEM='meson' allow_failures: - env: BUILD_SYSTEM='meson' @@ -69,22 +70,25 @@ script: - printf 'JOB_NUMBER=%s TRAVIS_BRANCH=%s TRAVIS_EVENT_TYPE=%s TRAVIS_PULL_REQUEST=%s\n' "${JOB_NUMBER}" "${TRAVIS_BRANCH}" "${TRAVIS_EVENT_TYPE}" "${TRAVIS_PULL_REQUEST}" - if [ "${BUILD_SYSTEM}" = meson ]; then - sudo apt-get install -qq python3.5 wget tree - && wget https://bootstrap.pypa.io/get-pip.py - && python3.5 get-pip.py --user + set -x; + sudo apt-get install -qq liblz4-dev valgrind tree + && curl -o get-pip.py 'https://bootstrap.pypa.io/get-pip.py' + && python3 get-pip.py --user && rm get-pip.py - && pip3.5 install --user meson ninja; - mkdir build/meson/build; - pushd "$_"; - meson --buildtype=debug -D with-contrib=true -D with-tests=true - -D with-contrib=true -D default_library=both .. + && pip3 install --user meson ninja + && unset CC CXX + && meson --buildtype=debug + -Dauto_features=enabled + -Dbuild_{programs,tests,contrib}=true + -Ddefault_library=both build/meson + builddir + && cd "$_" && ninja && DESTDIR=./staging ninja install && tree ./staging; - popd; - else - export FUZZERTEST=-T2mn; - export ZSTREAM_TESTTIME=-T2mn; - export DECODECORPUS_TESTTIME=-T1mn; - sh -c "${Cmd}" || travis_terminate 1; + travis_terminate "$?"; fi + - export FUZZERTEST=-T2mn; + export ZSTREAM_TESTTIME=-T2mn; + export DECODECORPUS_TESTTIME=-T1mn; + sh -c "${Cmd}" || travis_terminate 1; From 8d5252b68d651981e4dfdb535fde10854c140a47 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Fri, 30 Nov 2018 21:31:58 +0700 Subject: [PATCH 42/62] meson: Remove redundant join_paths function --- build/meson/contrib/gen_html/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/meson/contrib/gen_html/meson.build b/build/meson/contrib/gen_html/meson.build index a2fea9d52..cabff209d 100644 --- a/build/meson/contrib/gen_html/meson.build +++ b/build/meson/contrib/gen_html/meson.build @@ -24,6 +24,6 @@ zstd_manual_html = custom_target('zstd_manual.html', output : 'zstd_manual.html', command : [gen_html, zstd_version, - join_paths(meson.current_source_dir(), join_paths(zstd_rootdir, 'lib/zstd.h')), + join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h'), '@OUTPUT@'], install : false) From 519b2de6f7f8940aa4a8793db8f57f369070b962 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sat, 1 Dec 2018 09:11:26 +0700 Subject: [PATCH 43/62] Update README --- build/meson/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/meson/README.md b/build/meson/README.md index f5818cc69..dae503fef 100644 --- a/build/meson/README.md +++ b/build/meson/README.md @@ -1,5 +1,5 @@ -contrib/meson - Meson build system for zstandard -================================================ +Meson build system for zstandard +================================ Meson is a build system designed to optimize programmer productivity. It aims to do this by providing simple, out-of-the-box support for From cbf2a924dd504e155dfeb6f6bc9708cc8c2dc1d9 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sat, 1 Dec 2018 13:07:08 +0700 Subject: [PATCH 44/62] meson: Cleanup installing symlinks --- build/meson/programs/meson.build | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build index f538aa556..c97bc653d 100644 --- a/build/meson/programs/meson.build +++ b/build/meson/programs/meson.build @@ -79,32 +79,23 @@ install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'), install_dir: zstd_bindir) # ============================================================================= -# Program symlinks +# Programs and manpages installing # ============================================================================= -InstallSymlink_py = '../InstallSymlink.py' -meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdcat', zstd_bindir) -meson.add_install_script(InstallSymlink_py, 'zstd', 'unzstd', zstd_bindir) - -if use_multi_thread - meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdmt', zstd_bindir) -endif - -# ============================================================================= -# Manpages -# ============================================================================= - -zstd_man1_dir = join_paths(zstd_mandir, 'man1') - install_man(join_paths(zstd_rootdir, 'programs/zstd.1'), join_paths(zstd_rootdir, 'programs/zstdgrep.1'), join_paths(zstd_rootdir, 'programs/zstdless.1')) -# Meson automatically compresses manpages to gz format -# WARNING: This may fail on Windows. Test NEEDED. -meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdcat.1.gz', zstd_man1_dir) -meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'unzstd.1.gz', zstd_man1_dir) +InstallSymlink_py = '../InstallSymlink.py' +zstd_man1_dir = join_paths(zstd_mandir, 'man1') +man1_EXT = host_machine_os != os_windows ? '.1.gz' : '' + +foreach f : ['zstdcat', 'unzstd'] + meson.add_install_script(InstallSymlink_py, 'zstd', f, zstd_bindir) + meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, f + man1_EXT, zstd_man1_dir) +endforeach if use_multi_thread - meson.add_install_script(InstallSymlink_py, 'zstd.1.gz', 'zstdmt.1.gz', zstd_man1_dir) + meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdmt', zstd_bindir) + meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, 'zstdmt' + man1_EXT, zstd_man1_dir) endif From 7da18bc85a759f7b41ac754ef611a1024ee89b79 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sun, 2 Dec 2018 00:18:00 +0700 Subject: [PATCH 45/62] Add missed .1 extension for manpage [skip ci] --- build/meson/programs/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/meson/programs/meson.build b/build/meson/programs/meson.build index c97bc653d..333ad8e8b 100644 --- a/build/meson/programs/meson.build +++ b/build/meson/programs/meson.build @@ -88,7 +88,7 @@ install_man(join_paths(zstd_rootdir, 'programs/zstd.1'), InstallSymlink_py = '../InstallSymlink.py' zstd_man1_dir = join_paths(zstd_mandir, 'man1') -man1_EXT = host_machine_os != os_windows ? '.1.gz' : '' +man1_EXT = host_machine_os != os_windows ? '.1.gz' : '.1' foreach f : ['zstdcat', 'unzstd'] meson.add_install_script(InstallSymlink_py, 'zstd', f, zstd_bindir) From 38728b4518ff9f674ad9911f02301af73d7518ea Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sun, 2 Dec 2018 22:31:18 +0700 Subject: [PATCH 46/62] Use argparse instead of manually parsing [skip ci] --- build/meson/GetZstdLibraryVersion.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/build/meson/GetZstdLibraryVersion.py b/build/meson/GetZstdLibraryVersion.py index 5659a3283..53c61008a 100644 --- a/build/meson/GetZstdLibraryVersion.py +++ b/build/meson/GetZstdLibraryVersion.py @@ -11,11 +11,6 @@ import re import sys -def usage(): - print('usage: python3 GetZstdLibraryVersion.py ') - sys.exit(1) - - def find_version(filepath): version_file_data = None with open(filepath) as fd: @@ -29,14 +24,15 @@ def find_version(filepath): version_match = regex.search(version_file_data) if version_match: return version_match.groups() - raise RuntimeError("Unable to find version string.") + raise Exception("Unable to find version string.") def main(): - if len(sys.argv) < 2: - usage() - - filepath = sys.argv[1] + import argparse + parser = argparse.ArgumentParser(description='Print zstd version from lib/zstd.h') + parser.add_argument('file', help='path to lib/zstd.h') + args = parser.parse_args() + filepath = args.file version_tup = find_version(filepath) print('.'.join(version_tup)) From 23d751507e4b2ef99dc27cc6f691835b52dc9b50 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sun, 2 Dec 2018 22:33:25 +0700 Subject: [PATCH 47/62] meson: Use -werror build option instead of adding -Werror [skip ci] --- build/meson/meson.build | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build/meson/meson.build b/build/meson/meson.build index c36c9425a..2e1f9d06b 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -68,6 +68,7 @@ zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name()) # Built-in options use_debug = get_option('debug') +buildtype = get_option('buildtype') # Custom options debug_level = get_option('debug_level') @@ -132,7 +133,9 @@ add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c']) if [compiler_gcc, compiler_clang].contains(cc_id) common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ] if cc_id == compiler_clang - common_warning_flags += ['-Werror', '-Wconversion', '-Wno-sign-conversion', '-Wdocumentation'] + # Should use Meson's own --werror build option + #common_warning_flags += '-Werror' + common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation'] endif cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes']) cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags) From 838de084390dd612c33999ffa39b9b4dac1107bf Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sun, 2 Dec 2018 22:45:46 +0700 Subject: [PATCH 48/62] meson: Fix soversion --- build/meson/lib/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index c39a65530..bce6b7c56 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -111,7 +111,8 @@ libzstd = library('zstd', c_args: libzstd_c_args, dependencies: libzstd_deps, install: true, - soversion: zstd_libversion) + version: zstd_libversion, + soversion: '1') libzstd_dep = declare_dependency(link_with: libzstd, include_directories: libzstd_includes) From 437ec5f47fc60fd8372a5c6517ea36c38fc3955a Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Sun, 2 Dec 2018 22:48:11 +0700 Subject: [PATCH 49/62] meson helper: Use Python conventional name for naming function [skip ci] --- build/meson/InstallSymlink.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build/meson/InstallSymlink.py b/build/meson/InstallSymlink.py index d7b1e5a0b..6dd151530 100644 --- a/build/meson/InstallSymlink.py +++ b/build/meson/InstallSymlink.py @@ -21,7 +21,7 @@ def mkdir_p(path, dir_mode=0o777): raise -def InstallSymlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777): +def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777): if not os.path.exists(install_dir): mkdir_p(install_dir, dir_mode) if not os.path.isdir(install_dir): @@ -38,8 +38,7 @@ def InstallSymlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777): def main(): import argparse parser = argparse.ArgumentParser(description='Install a symlink.\n', - usage='usage: InstallSymlink.py [-h] [-d] [-m MODE] src dst ' - 'install_dir\n\n' + usage='InstallSymlink.py [-h] [-d] [-m MODE] src dst install_dir\n\n' 'example:\n' '\tInstallSymlink.py libcrypto.so.1.0.0 libcrypt.so ' '/usr/lib/x86_64-linux-gnu False') @@ -65,7 +64,7 @@ def main(): install_dir = DESTDIR + install_dir if os.path.isabs(install_dir) \ else os.path.join(DESTDIR, install_dir) - InstallSymlink(src, dst, install_dir, dst_is_dir, dir_mode) + install_symlink(src, dst, install_dir, dst_is_dir, dir_mode) if __name__ == '__main__': From 65507666bb468c4708359b8dfa7146e7113d624a Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Mon, 3 Dec 2018 00:36:40 +0700 Subject: [PATCH 50/62] Use -Dlegacy_level build option to control ZSTD_LEGACY_SUPPORT macro in test --- build/meson/tests/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build index c8772e761..e35575d00 100644 --- a/build/meson/tests/meson.build +++ b/build/meson/tests/meson.build @@ -110,7 +110,8 @@ invalidDictionaries = executable('invalidDictionaries', legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')] legacy = executable('legacy', legacy_sources, - c_args: '-DZSTD_LEGACY_SUPPORT=4', + # Use -Dlegacy_level build option to control it + #c_args: '-DZSTD_LEGACY_SUPPORT=4', dependencies: [ libzstd_dep ], install: false) From 25311d24c6565f027f25e5a81e55e78be816b4db Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Mon, 3 Dec 2018 00:57:10 +0700 Subject: [PATCH 51/62] meson: Use clang for faster build --- .travis.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1ed148d0b..8cb40b1e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,18 +72,17 @@ script: - if [ "${BUILD_SYSTEM}" = meson ]; then set -x; sudo apt-get install -qq liblz4-dev valgrind tree - && curl -o get-pip.py 'https://bootstrap.pypa.io/get-pip.py' - && python3 get-pip.py --user - && rm get-pip.py + && curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' + && python3 ~/get-pip.py --user && pip3 install --user meson ninja - && unset CC CXX + && export CC=clang CXX=clang++ && meson --buildtype=debug - -Dauto_features=enabled - -Dbuild_{programs,tests,contrib}=true - -Ddefault_library=both build/meson - builddir + -Db_lundef=false + -Dauto_features=enabled + -Dbuild_{programs,tests,contrib}=true + -Ddefault_library=both + build/meson builddir && cd "$_" - && ninja && DESTDIR=./staging ninja install && tree ./staging; travis_terminate "$?"; From 9d6cf606f8b0cdb2f2101484f44ca9bff24eb47d Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Mon, 3 Dec 2018 03:23:09 +0700 Subject: [PATCH 52/62] meson: Update tests timeout to run properly --- build/meson/tests/meson.build | 54 +++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build index e35575d00..aa9cd86f4 100644 --- a/build/meson/tests/meson.build +++ b/build/meson/tests/meson.build @@ -173,22 +173,46 @@ if host_machine_os != os_windows timeout: 600) # Timeout should work on HDD drive endif -test('test-fullbench-1', fullbench, args: ['-i1'], - depends: [fullbench, datagen]) -test('test-fullbench-2', fullbench, args: ['-i1', '-P0'], - depends: [fullbench, datagen]) +test('test-fullbench-1', + fullbench, + args: ['-i1'], + depends: [datagen], + timeout: 60) +test('test-fullbench-2', + fullbench, + args: ['-i1', '-P0'], + depends: [datagen], + timeout: 60) if use_zlib - test('test-fuzzer', fuzzer, args: ['-v', FUZZERTEST] + FUZZER_FLAGS) + test('test-fuzzer', + fuzzer, + args: ['-v', FUZZERTEST] + FUZZER_FLAGS, + timeout: 240) endif -test('test-zbuff', zbufftest, args: [ZSTREAM_TESTTIME]) -test('test-zstream-1', zstreamtest, args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS) -test('test-zstream-2', zstreamtest, args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS) -test('test-zstream-3', zstreamtest, args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS) -test('test-longmatch', longmatch) -test('test-invalidDictionaries', invalidDictionaries) -test('test-symbols', symbols) -test('test-legacy', legacy) -test('test-decodecorpus', decodecorpus, args: ['-t', DECODECORPUS_TESTTIME]) -test('test-poolTests', poolTests) +test('test-zbuff', + zbufftest, + args: [ZSTREAM_TESTTIME], + timeout: 120) +test('test-zstream-1', + zstreamtest, + args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS, + timeout: 120) +test('test-zstream-2', + zstreamtest, + args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS, + timeout: 120) +test('test-zstream-3', + zstreamtest, + args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS, + timeout: 120) +test('test-longmatch', longmatch, timeout: 36) +test('test-invalidDictionaries', invalidDictionaries) # should be fast +test('test-symbols', symbols) # should be fast +test('test-legacy', legacy) # should be fast +test('test-decodecorpus', + decodecorpus, + args: ['-t', DECODECORPUS_TESTTIME], + timeout: 60) +test('test-poolTests', poolTests) # should be fast From f8975236555a2712e75783b80d1019998f54b118 Mon Sep 17 00:00:00 2001 From: Lzu Tao Date: Mon, 3 Dec 2018 11:02:42 +0700 Subject: [PATCH 53/62] meson: Update usage of InstallSymlink helper --- build/meson/InstallSymlink.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build/meson/InstallSymlink.py b/build/meson/InstallSymlink.py index 6dd151530..9c68cb33e 100644 --- a/build/meson/InstallSymlink.py +++ b/build/meson/InstallSymlink.py @@ -29,19 +29,19 @@ def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777): new_dst = os.path.join(install_dir, dst) if os.path.islink(new_dst) and os.readlink(new_dst) == src: - print('File exists: %r -> %r' % (dst, src)) + print('File exists: {!r} -> {!r}'.format(new_dst, src)) return - print('Installing symlink %r -> %r' % (new_dst, src)) + print('Installing symlink {!r} -> {!r}'.format(new_dst, src)) os.symlink(src, new_dst, dst_is_dir) def main(): import argparse - parser = argparse.ArgumentParser(description='Install a symlink.\n', + parser = argparse.ArgumentParser(description='Install a symlink', usage='InstallSymlink.py [-h] [-d] [-m MODE] src dst install_dir\n\n' 'example:\n' - '\tInstallSymlink.py libcrypto.so.1.0.0 libcrypt.so ' - '/usr/lib/x86_64-linux-gnu False') + '\tInstallSymlink.py dash sh /bin\n' + '\tDESTDIR=./staging InstallSymlink.py dash sh /bin') parser.add_argument('src', help='target to link') parser.add_argument('dst', help='link name') parser.add_argument('install_dir', help='installation directory') From 7ef7dc561aaee98166fa6a9a72314a86dc9e2f3e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 3 Dec 2018 15:46:55 -0800 Subject: [PATCH 54/62] check availability of --color=never command on grep and egrep before applying them. Fixes #1436 --- Makefile | 7 ++++++- lib/Makefile | 7 ++++++- programs/Makefile | 14 ++++++++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c63db80e9..f3a1bc7ed 100644 --- a/Makefile +++ b/Makefile @@ -129,7 +129,12 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr HOST_OS = POSIX CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release -EGREP = egrep --color=never +HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) +EGREP_OPTIONS ?= +ifeq ($HAVE_COLORNEVER, 1) +EGREP_OPTIONS += --color=never +endif +EGREP = egrep $(EGREP_OPTIONS) # Print a two column output of targets and their description. To add a target description, put a # comment in the Makefile with the format "## : ". For example: diff --git a/lib/Makefile b/lib/Makefile index b39786cf2..3fddf4fcd 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -31,7 +31,12 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) -GREP = grep --color=never +HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) +GREP_OPTIONS ?= +ifeq ($HAVE_COLORNEVER, 1) +GREP_OPTIONS += --color=never +endif +GREP = grep $(GREP_OPTIONS) ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c)) ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c)) diff --git a/programs/Makefile b/programs/Makefile index 6249e5921..d1910fbb4 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -29,7 +29,12 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT)) ZSTD_VERSION = $(LIBVER) -GREP = grep --color=never +HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) +GREP_OPTIONS ?= +ifeq ($HAVE_COLORNEVER, 1) +GREP_OPTIONS += --color=never +endif +GREP = grep $(GREP_OPTIONS) ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1) ALIGN_LOOP = -falign-loops=32 @@ -275,7 +280,12 @@ preview-man: clean-man man #----------------------------------------------------------------------------- ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) -EGREP = egrep --color=never +HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) +EGREP_OPTIONS ?= +ifeq ($HAVE_COLORNEVER, 1) +EGREP_OPTIONS += --color=never +endif +EGREP = egrep $(EGREP_OPTIONS) # Print a two column output of targets and their description. To add a target description, put a # comment in the Makefile with the format "## : ". For example: From f67da6128173c9ac9d4d02cf469b2d1b2f24eeef Mon Sep 17 00:00:00 2001 From: Mitchell Grenier Date: Thu, 6 Dec 2018 14:46:48 -0800 Subject: [PATCH 55/62] Add buck build instruction to the readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index c0b550572..146264cf5 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,10 @@ Going into `build` directory, you will find additional possibilities: - Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`, which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution. +#### Buck + +You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo. +The output binary will be in `buck-out/gen/programs/`. ### Status From 0f52b0caadde23a4876c87c6ba053116d237fb09 Mon Sep 17 00:00:00 2001 From: Hung Dang Date: Thu, 6 Dec 2018 22:42:19 -0500 Subject: [PATCH 56/62] Refactor examples to avoid code duplication. --- examples/dictionary_compression.c | 60 +------ examples/dictionary_decompression.c | 45 +----- examples/multiple_streaming_compression.c | 50 +----- examples/simple_compression.c | 68 +------- examples/simple_decompression.c | 46 +----- examples/streaming_compression.c | 51 +----- examples/streaming_decompression.c | 48 +----- examples/utils.h | 181 ++++++++++++++++++++++ 8 files changed, 189 insertions(+), 360 deletions(-) create mode 100644 examples/utils.h diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c index 97bf8cb5e..511b35676 100644 --- a/examples/dictionary_compression.c +++ b/examples/dictionary_compression.c @@ -7,71 +7,13 @@ * in the COPYING file in the root directory of this source tree). * You may select, at your option, one of the above-listed licenses. */ - - #include // malloc, exit #include // printf #include // strerror #include // errno #include // stat #include // presumes zstd library is installed - - -static off_t fsize_orDie(const char *filename) -{ - struct stat st; - if (stat(filename, &st) == 0) return st.st_size; - /* error */ - perror(filename); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(2); -} - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc"); - exit(3); -} - -static void* loadFile_orDie(const char* fileName, size_t* size) -{ - off_t const buffSize = fsize_orDie(fileName); - FILE* const inFile = fopen_orDie(fileName, "rb"); - void* const buffer = malloc_orDie(buffSize); - size_t const readSize = fread(buffer, 1, buffSize, inFile); - if (readSize != (size_t)buffSize) { - fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); - exit(4); - } - fclose(inFile); - *size = buffSize; - return buffer; -} - -static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize) -{ - FILE* const oFile = fopen_orDie(fileName, "wb"); - size_t const wSize = fwrite(buff, 1, buffSize, oFile); - if (wSize != (size_t)buffSize) { - fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno)); - exit(5); - } - if (fclose(oFile)) { - perror(fileName); - exit(6); - } -} +#include "utils.h" /* createDict() : `dictFileName` is supposed to have been created using `zstd --train` */ diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index 07e6e24c6..69f56d56b 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -17,49 +17,7 @@ #include // stat #define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize #include // presumes zstd library is installed - - -static off_t fsize_orDie(const char *filename) -{ - struct stat st; - if (stat(filename, &st) == 0) return st.st_size; - /* error */ - perror(filename); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(2); -} - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc"); - exit(3); -} - -static void* loadFile_orDie(const char* fileName, size_t* size) -{ - off_t const buffSize = fsize_orDie(fileName); - FILE* const inFile = fopen_orDie(fileName, "rb"); - void* const buffer = malloc_orDie(buffSize); - size_t const readSize = fread(buffer, 1, buffSize, inFile); - if (readSize != (size_t)buffSize) { - fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); - exit(4); - } - fclose(inFile); - *size = buffSize; - return buffer; -} +#include "utils.h" /* createDict() : `dictFileName` is supposed to have been created using `zstd --train` */ @@ -74,7 +32,6 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName) return ddict; } - static void decompress(const char* fname, const ZSTD_DDict* ddict) { size_t cSize; diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c index 4308a2e4d..442ff40ae 100644 --- a/examples/multiple_streaming_compression.c +++ b/examples/multiple_streaming_compression.c @@ -20,53 +20,7 @@ #include // errno #define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being #include // presumes zstd library is installed - - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc:"); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(3); -} - -static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) -{ - size_t const readSize = fread(buffer, 1, sizeToRead, file); - if (readSize == sizeToRead) return readSize; /* good */ - if (feof(file)) return readSize; /* good, reached end of file */ - /* error */ - perror("fread"); - exit(4); -} - -static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) -{ - size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); - if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ - /* error */ - perror("fwrite"); - exit(5); -} - -static size_t fclose_orDie(FILE* file) -{ - if (!fclose(file)) return 0; - /* error */ - perror("fclose"); - exit(6); -} - +#include "utils.h" typedef struct { void* buffIn; @@ -95,7 +49,6 @@ static void freeResources(resources ress) free(ress.buffOut); } - static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel) { FILE* const fin = fopen_orDie(fname, "rb"); @@ -125,7 +78,6 @@ static void compressFile_orDie(resources ress, const char* fname, const char* ou fclose_orDie(fin); } - int main(int argc, const char** argv) { const char* const exeName = argv[0]; diff --git a/examples/simple_compression.c b/examples/simple_compression.c index 9ade424a2..0193dd40f 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -8,78 +8,13 @@ * You may select, at your option, one of the above-listed licenses. */ - - #include // malloc, free, exit #include // fprintf, perror, fopen, etc. #include // strlen, strcat, memset, strerror #include // errno #include // stat #include // presumes zstd library is installed - - -static off_t fsize_orDie(const char *filename) -{ - struct stat st; - if (stat(filename, &st) == 0) return st.st_size; - /* error */ - perror(filename); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(2); -} - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror(NULL); - exit(3); -} - -static void* loadFile_orDie(const char* fileName, size_t* size) -{ - off_t const fileSize = fsize_orDie(fileName); - size_t const buffSize = (size_t)fileSize; - if ((off_t)buffSize < fileSize) { /* narrowcast overflow */ - fprintf(stderr, "%s : filesize too large \n", fileName); - exit(4); - } - FILE* const inFile = fopen_orDie(fileName, "rb"); - void* const buffer = malloc_orDie(buffSize); - size_t const readSize = fread(buffer, 1, buffSize, inFile); - if (readSize != (size_t)buffSize) { - fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); - exit(5); - } - fclose(inFile); /* can't fail, read only */ - *size = buffSize; - return buffer; -} - - -static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize) -{ - FILE* const oFile = fopen_orDie(fileName, "wb"); - size_t const wSize = fwrite(buff, 1, buffSize, oFile); - if (wSize != (size_t)buffSize) { - fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno)); - exit(6); - } - if (fclose(oFile)) { - perror(fileName); - exit(7); - } -} - +#include "utils.h" static void compress_orDie(const char* fname, const char* oname) { @@ -103,7 +38,6 @@ static void compress_orDie(const char* fname, const char* oname) free(cBuff); } - static char* createOutFilename_orDie(const char* filename) { size_t const inL = strlen(filename); diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index c1818a95c..ee055dd67 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -15,50 +15,7 @@ #include // stat #define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize #include // presumes zstd library is installed - - -static off_t fsize_orDie(const char *filename) -{ - struct stat st; - if (stat(filename, &st) == 0) return st.st_size; - /* error */ - fprintf(stderr, "stat: %s : %s \n", filename, strerror(errno)); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - fprintf(stderr, "fopen: %s : %s \n", filename, strerror(errno)); - exit(2); -} - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size + !size); /* avoid allocating size of 0 : may return NULL (implementation dependent) */ - if (buff) return buff; - /* error */ - fprintf(stderr, "malloc: %s \n", strerror(errno)); - exit(3); -} - -static void* loadFile_orDie(const char* fileName, size_t* size) -{ - off_t const buffSize = fsize_orDie(fileName); - FILE* const inFile = fopen_orDie(fileName, "rb"); - void* const buffer = malloc_orDie(buffSize); - size_t const readSize = fread(buffer, 1, buffSize, inFile); - if (readSize != (size_t)buffSize) { - fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); - exit(4); - } - fclose(inFile); /* can't fail (read only) */ - *size = buffSize; - return buffer; -} - +#include "utils.h" static void decompress(const char* fname) { @@ -90,7 +47,6 @@ static void decompress(const char* fname) free(cBuff); } - int main(int argc, const char** argv) { const char* const exeName = argv[0]; diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index 9287ff398..e056f86f0 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -13,53 +13,7 @@ #include // fprintf, perror, feof, fopen, etc. #include // strlen, memset, strcat #include // presumes zstd library is installed - - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc:"); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(3); -} - -static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) -{ - size_t const readSize = fread(buffer, 1, sizeToRead, file); - if (readSize == sizeToRead) return readSize; /* good */ - if (feof(file)) return readSize; /* good, reached end of file */ - /* error */ - perror("fread"); - exit(4); -} - -static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) -{ - size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); - if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ - /* error */ - perror("fwrite"); - exit(5); -} - -static size_t fclose_orDie(FILE* file) -{ - if (!fclose(file)) return 0; - /* error */ - perror("fclose"); - exit(6); -} - +#include "utils.h" static void compressFile_orDie(const char* fname, const char* outName, int cLevel) { @@ -102,8 +56,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve ZSTD_freeCStream(cstream); fclose_orDie(fout); - fclose_orDie(fin); - free(buffIn); + fclose_orDie(fin); free(buffIn); free(buffOut); } diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c index 504a5e316..13c3c509e 100644 --- a/examples/streaming_decompression.c +++ b/examples/streaming_decompression.c @@ -14,53 +14,7 @@ #include // strerror #include // errno #include // presumes zstd library is installed - - -static void* malloc_orDie(size_t size) -{ - void* const buff = malloc(size); - if (buff) return buff; - /* error */ - perror("malloc:"); - exit(1); -} - -static FILE* fopen_orDie(const char *filename, const char *instruction) -{ - FILE* const inFile = fopen(filename, instruction); - if (inFile) return inFile; - /* error */ - perror(filename); - exit(3); -} - -static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) -{ - size_t const readSize = fread(buffer, 1, sizeToRead, file); - if (readSize == sizeToRead) return readSize; /* good */ - if (feof(file)) return readSize; /* good, reached end of file */ - /* error */ - perror("fread"); - exit(4); -} - -static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) -{ - size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); - if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ - /* error */ - perror("fwrite"); - exit(5); -} - -static size_t fclose_orDie(FILE* file) -{ - if (!fclose(file)) return 0; - /* error */ - perror("fclose"); - exit(6); -} - +#include "utils.h" static void decompressFile_orDie(const char* fname) { diff --git a/examples/utils.h b/examples/utils.h new file mode 100644 index 000000000..55a329e23 --- /dev/null +++ b/examples/utils.h @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + * This header file has common utility functions used in examples. + */ +#ifndef UTILS_H +#define UTILS_H + +#include // malloc, free, exit +#include // fprintf, perror, fopen, etc. +#include // strlen, strcat, memset, strerror +#include // errno +#include // stat + +/* + * Define the returned error code from utility functions. + */ +typedef enum { + ERROR_fsize = 1, + ERROR_fopen = 2, + ERROR_fclose = 3, + ERROR_fread = 4, + ERROR_fwrite = 5, + ERROR_loadFile = 6, + ERROR_saveFile = 7, + ERROR_malloc = 8, + ERROR_largeFile = 9, +} UTILS_ErrorCode; + +/*! fsize_orDie() : + * Get the size of a given file path. + * + * @return The size of a given file path. + */ +static off_t fsize_orDie(const char *filename) +{ + struct stat st; + if (stat(filename, &st) == 0) return st.st_size; + /* error */ + perror(filename); + exit(ERROR_fsize); +} + +/*! fopen_orDie() : + * Open a file using given file path and open option. + * + * @return If successful this function will return a FILE pointer to an + * opened file otherwise it sends an error to stderr and exits. + */ +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(ERROR_fopen); +} + +/*! fclose_orDie() : + * Close an opened file using given FILE pointer. + */ +static void fclose_orDie(FILE* file) +{ + if (!fclose(file)) { return; }; + /* error */ + perror("fclose"); + exit(ERROR_fclose); +} + +/*! fread_orDie() : + * + * Read sizeToRead bytes from a given file, storing them at the + * location given by buffer. + * + * @return The number of bytes read. + */ +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(ERROR_fread); +} + +/*! fwrite_orDie() : + * + * Write sizeToWrite bytes to a file pointed to by file, obtaining + * them from a location given by buffer. + * + * Note: This function will send an error to stderr and exit if it + * cannot write data to the given file pointer. + * + * @return The number of bytes written. + */ +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(ERROR_fwrite); +} + +/*! malloc_orDie() : + * Allocate memory. + * + * @return If successful this function returns a pointer to allo- + * cated memory. If there is an error, this function will send that + * error to stderr and exit. + */ +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc"); + exit(ERROR_malloc); +} + +/*! loadFile_orDie() : + * Read size bytes from a file. + * + * Note: This function will send an error to stderr and exit if it + * cannot read data from the given file path. + * + * @return If successful this function will return a pointer to read + * data otherwise it will printout an error to stderr and exit. + */ +static void* loadFile_orDie(const char* fileName, size_t* size) +{ + off_t const fileSize = fsize_orDie(fileName); + size_t const buffSize = (size_t)fileSize; + if ((off_t)buffSize < fileSize) { /* narrowcast overflow */ + fprintf(stderr, "%s : filesize too large \n", fileName); + exit(ERROR_largeFile); + } + FILE* const inFile = fopen_orDie(fileName, "rb"); + void* const buffer = malloc_orDie(buffSize); + size_t const readSize = fread(buffer, 1, buffSize, inFile); + if (readSize != (size_t)buffSize) { + fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); + exit(ERROR_fread); + } + fclose(inFile); /* can't fail, read only */ + *size = buffSize; + return buffer; +} + +/*! saveFile_orDie() : + * + * Save buffSize bytes to a given file path, obtaining them from a location pointed + * to by buff. + * + * Note: This function will send an error to stderr and exit if it + * cannot write to a given file. + */ +static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize) +{ + FILE* const oFile = fopen_orDie(fileName, "wb"); + size_t const wSize = fwrite(buff, 1, buffSize, oFile); + if (wSize != (size_t)buffSize) { + fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno)); + exit(ERROR_fwrite); + } + if (fclose(oFile)) { + perror(fileName); + exit(ERROR_fclose); + } +} + +#endif From c226a7b9f32c0ca02fed147a6152e09c1fc8063f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 10 Dec 2018 17:33:49 -0800 Subject: [PATCH 57/62] fixed ZSTD_compress2() as suggested by @terrelln --- lib/compress/zstd_compress.c | 26 ++++++++++++++------------ lib/zstd.h | 2 ++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3b84e1b5c..3d6ed47eb 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -4060,19 +4060,21 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t oPos = 0; - size_t iPos = 0; - size_t const result = ZSTD_compressStream2_simpleArgs(cctx, - dst, dstCapacity, &oPos, - src, srcSize, &iPos, - ZSTD_e_end); - assert(iPos == srcSize); - if (ZSTD_isError(result)) return result; - if (result != 0) { /* compression not completed, due to lack of output space */ - assert(oPos == dstCapacity); - return ERROR(dstSize_tooSmall); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + { size_t oPos = 0; + size_t iPos = 0; + size_t const result = ZSTD_compressStream2_simpleArgs(cctx, + dst, dstCapacity, &oPos, + src, srcSize, &iPos, + ZSTD_e_end); + if (ZSTD_isError(result)) return result; + assert(iPos == srcSize); + if (result != 0) { /* compression not completed, due to lack of output space */ + assert(oPos == dstCapacity); + return ERROR(dstSize_tooSmall); + } + return oPos; } - return oPos; } /*====== Finalize ======*/ diff --git a/lib/zstd.h b/lib/zstd.h index 72289d99d..0de81f3ac 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -750,6 +750,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); /*! ZSTD_compress2() : * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() * - The function is always blocking, returns when compression is completed. * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. From 5a1e01e5f15302cd3aa33905645ce97954028be8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 10 Dec 2018 17:36:20 -0800 Subject: [PATCH 58/62] clarified experimentalParam --- lib/zstd.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index 0de81f3ac..3e977aa09 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -620,12 +620,13 @@ typedef enum { * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. */ ZSTD_c_experimentalParam1=500, ZSTD_c_experimentalParam2=10, ZSTD_c_experimentalParam3=1000, - ZSTD_c_experimentalParam4 + ZSTD_c_experimentalParam4=1001 } ZSTD_cParameter; From 3619c34399acc8f68eadd6a2d88b7c47b2f6deca Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 10 Dec 2018 17:42:35 -0800 Subject: [PATCH 59/62] fix assert position within ZSTD_compress2() --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3d6ed47eb..f05b5e107 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -4068,11 +4068,11 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, src, srcSize, &iPos, ZSTD_e_end); if (ZSTD_isError(result)) return result; - assert(iPos == srcSize); if (result != 0) { /* compression not completed, due to lack of output space */ assert(oPos == dstCapacity); return ERROR(dstSize_tooSmall); } + assert(iPos == srcSize); /* all input is expected consumed */ return oPos; } } From 52b94f902c0b9d9c56831811b5772989360c5f07 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 11 Dec 2018 12:08:21 -0800 Subject: [PATCH 60/62] add clarification for ZSTD_CCtx_setPledgedSrcSize() as requested in #1391 --- doc/zstd_manual.html | 21 ++++++++++++++------- lib/zstd.h | 14 +++++++++----- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 92566e1dd..3db037c15 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -487,8 +487,9 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); /* frame parameters */ ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) - * Content size must be known at the beginning of compression, - * it is provided using ZSTD_CCtx_setPledgedSrcSize() */ + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ @@ -524,12 +525,13 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. - * note : never ever use experimentalParam? names directly + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. */ ZSTD_c_experimentalParam1=500, ZSTD_c_experimentalParam2=10, ZSTD_c_experimentalParam3=1000, - ZSTD_c_experimentalParam4 + ZSTD_c_experimentalParam4=1001 } ZSTD_cParameter; 
typedef struct {
@@ -562,14 +564,17 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
 
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
 
  Total input data size to be compressed as a single frame.
-  This value will be controlled at end of frame, and trigger an error if not respected.
+  Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
+  This value will also be controlled at end of frame, and trigger an error if not respected.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
            In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
            ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
   Note 2 : pledgedSrcSize is only valid once, for the next frame.
-           It's discarded at the end of the frame.
-  Note 3 : If all data is provided and consumed in a single round,
+           It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
+  Note 3 : Whenever all input data is provided and consumed in a single round,
+           for example with ZSTD_compress2(),
+           or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
            this value is automatically overriden by srcSize instead.
  
 


@@ -652,6 +657,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
                        void* dst, size_t dstCapacity,
                  const void* src, size_t srcSize);
 
  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+  ZSTD_compress2() always starts a new frame.
+  Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
   - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
   - The function is always blocking, returns when compression is completed.
   Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
diff --git a/lib/zstd.h b/lib/zstd.h
index 3e977aa09..198500e55 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -583,8 +583,9 @@ typedef enum {
 
     /* frame parameters */
     ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
-                              * Content size must be known at the beginning of compression,
-                              * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
+                              * Content size must be known at the beginning of compression.
+                              * This is automatically the case when using ZSTD_compress2(),
+                              * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
     ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_c_dictIDFlag=202,   /* When applicable, dictionary's ID is written into frame header (default:1) */
 
@@ -660,14 +661,17 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
 
 /*! ZSTD_CCtx_setPledgedSrcSize() :
  *  Total input data size to be compressed as a single frame.
- *  This value will be controlled at end of frame, and trigger an error if not respected.
+ *  Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
+ *  This value will also be controlled at end of frame, and trigger an error if not respected.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
  *           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
  *           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
  *  Note 2 : pledgedSrcSize is only valid once, for the next frame.
- *           It's discarded at the end of the frame.
- *  Note 3 : If all data is provided and consumed in a single round,
+ *           It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
+ *  Note 3 : Whenever all input data is provided and consumed in a single round,
+ *           for example with ZSTD_compress2(),
+ *           or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
  *           this value is automatically overriden by srcSize instead.
  */
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);

From fcfea057a16162de07fe1eaef4772d20f61205a4 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 3 Dec 2018 20:06:26 -0800
Subject: [PATCH 61/62] [regression] add more methods

---
 tests/regression/config.c    |  90 +++++--
 tests/regression/config.h    |  10 +-
 tests/regression/method.c    | 311 ++++++++++++++++++++++--
 tests/regression/results.csv | 451 +++++++++++++++++++++++++++--------
 4 files changed, 718 insertions(+), 144 deletions(-)

diff --git a/tests/regression/config.c b/tests/regression/config.c
index 15713d7eb..4a792e9c6 100644
--- a/tests/regression/config.c
+++ b/tests/regression/config.c
@@ -13,7 +13,7 @@
 /* Define a config for each fast level we want to test with. */
 #define FAST_LEVEL(x)                                               \
     param_value_t const level_fast##x##_param_values[] = {          \
-        {.param = ZSTD_p_compressionLevel, .value = (unsigned)-x},  \
+        {.param = ZSTD_c_compressionLevel, .value = -x},            \
     };                                                              \
     config_t const level_fast##x = {                                \
         .name = "level -" #x,                                       \
@@ -28,23 +28,22 @@
     };
 
 /* Define a config for each level we want to test with. */
-#define LEVEL(x)                                                  \
-    param_value_t const level_##x##_param_values[] = {            \
-        {.param = ZSTD_p_compressionLevel, .value = (unsigned)x}, \
-    };                                                            \
-    config_t const level_##x = {                                  \
-        .name = "level " #x,                                      \
-        .cli_args = "-" #x,                                       \
-        .param_values = PARAM_VALUES(level_##x##_param_values),   \
-    };                                                            \
-    config_t const level_##x##_dict = {                           \
-        .name = "level " #x " with dict",                         \
-        .cli_args = "-" #x,                                       \
-        .param_values = PARAM_VALUES(level_##x##_param_values),   \
-        .use_dictionary = 1,                                      \
+#define LEVEL(x)                                                \
+    param_value_t const level_##x##_param_values[] = {          \
+        {.param = ZSTD_c_compressionLevel, .value = x},         \
+    };                                                          \
+    config_t const level_##x = {                                \
+        .name = "level " #x,                                    \
+        .cli_args = "-" #x,                                     \
+        .param_values = PARAM_VALUES(level_##x##_param_values), \
+    };                                                          \
+    config_t const level_##x##_dict = {                         \
+        .name = "level " #x " with dict",                       \
+        .cli_args = "-" #x,                                     \
+        .param_values = PARAM_VALUES(level_##x##_param_values), \
+        .use_dictionary = 1,                                    \
     };
 
-
 #define PARAM_VALUES(pv) \
     { .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) }
 
@@ -56,7 +55,7 @@
 static config_t no_pledged_src_size = {
     .name = "no source size",
     .cli_args = "",
-    .param_values = {.data = NULL, .size = 0},
+    .param_values = PARAM_VALUES(level_0_param_values),
     .no_pledged_src_size = 1,
 };
 
@@ -82,8 +81,61 @@ int config_get_level(config_t const* config) {
     param_values_t const params = config->param_values;
     size_t i;
     for (size_t i = 0; i < params.size; ++i) {
-        if (params.data[i].param == ZSTD_p_compressionLevel)
-            return params.data[i].value;
+        if (params.data[i].param == ZSTD_c_compressionLevel)
+            return (int)params.data[i].value;
     }
     return CONFIG_NO_LEVEL;
 }
+
+ZSTD_parameters config_get_zstd_params(
+    config_t const* config,
+    uint64_t srcSize,
+    size_t dictSize) {
+    ZSTD_parameters zparams = {};
+    param_values_t const params = config->param_values;
+    int level = config_get_level(config);
+    if (level == CONFIG_NO_LEVEL)
+        level = 3;
+    zparams = ZSTD_getParams(
+        level,
+        config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : srcSize,
+        dictSize);
+    for (size_t i = 0; i < params.size; ++i) {
+        unsigned const value = params.data[i].value;
+        switch (params.data[i].param) {
+            case ZSTD_c_contentSizeFlag:
+                zparams.fParams.contentSizeFlag = value;
+                break;
+            case ZSTD_c_checksumFlag:
+                zparams.fParams.checksumFlag = value;
+                break;
+            case ZSTD_c_dictIDFlag:
+                zparams.fParams.noDictIDFlag = !value;
+                break;
+            case ZSTD_c_windowLog:
+                zparams.cParams.windowLog = value;
+                break;
+            case ZSTD_c_chainLog:
+                zparams.cParams.chainLog = value;
+                break;
+            case ZSTD_c_hashLog:
+                zparams.cParams.hashLog = value;
+                break;
+            case ZSTD_c_searchLog:
+                zparams.cParams.searchLog = value;
+                break;
+            case ZSTD_c_minMatch:
+                zparams.cParams.minMatch = value;
+                break;
+            case ZSTD_c_targetLength:
+                zparams.cParams.targetLength = value;
+                break;
+            case ZSTD_c_compressionStrategy:
+                zparams.cParams.strategy = (ZSTD_strategy)value;
+                break;
+            default:
+                break;
+        }
+    }
+    return zparams;
+}
diff --git a/tests/regression/config.h b/tests/regression/config.h
index 164f48b55..3cd0308a0 100644
--- a/tests/regression/config.h
+++ b/tests/regression/config.h
@@ -20,7 +20,7 @@
 
 typedef struct {
     ZSTD_cParameter param;
-    unsigned value;
+    int value;
 } param_value_t;
 
 typedef struct {
@@ -70,6 +70,14 @@ int config_skip_data(config_t const* config, data_t const* data);
  */
 int config_get_level(config_t const* config);
 
+/**
+ * Returns the compression parameters specified by the config.
+ */
+ZSTD_parameters config_get_zstd_params(
+    config_t const* config,
+    uint64_t srcSize,
+    size_t dictSize);
+
 /**
  * The NULL-terminated list of configs.
  */
diff --git a/tests/regression/method.c b/tests/regression/method.c
index 3fedcbe78..73d8bbf59 100644
--- a/tests/regression/method.c
+++ b/tests/regression/method.c
@@ -15,6 +15,8 @@
 
 #include 
 
+#define MIN(x, y) ((x) < (y) ? (x) : (y))
+
 static char const* g_zstdcli = NULL;
 
 void method_set_zstdcli(char const* zstdcli) {
@@ -35,6 +37,7 @@ void method_set_zstdcli(char const* zstdcli) {
 typedef struct {
     method_state_t base;
     data_buffers_t inputs; /**< The input buffer for each file. */
+    data_buffer_t dictionary; /**< The dictionary. */
     data_buffer_t compressed; /**< The compressed data buffer. */
     data_buffer_t decompressed; /**< The decompressed data buffer. */
 } buffer_state_t;
@@ -54,6 +57,7 @@ static method_state_t* buffer_state_create(data_t const* data) {
         return NULL;
     state->base.data = data;
     state->inputs = data_buffers_get(data);
+    state->dictionary = data_buffer_get_dict(data);
     size_t const max_size = buffers_max_size(state->inputs);
     state->compressed = data_buffer_create(ZSTD_compressBound(max_size));
     state->decompressed = data_buffer_create(max_size);
@@ -67,7 +71,9 @@ static void buffer_state_destroy(method_state_t* base) {
     free(state);
 }
 
-static int buffer_state_bad(buffer_state_t const* state) {
+static int buffer_state_bad(
+    buffer_state_t const* state,
+    config_t const* config) {
     if (state == NULL) {
         fprintf(stderr, "buffer_state_t is NULL\n");
         return 1;
@@ -77,13 +83,17 @@ static int buffer_state_bad(buffer_state_t const* state) {
         fprintf(stderr, "buffer state allocation failure\n");
         return 1;
     }
+    if (config->use_dictionary && state->dictionary.data == NULL) {
+        fprintf(stderr, "dictionary loading failed\n");
+        return 1;
+    }
     return 0;
 }
 
 static result_t simple_compress(method_state_t* base, config_t const* config) {
     buffer_state_t* state = container_of(base, buffer_state_t, base);
 
-    if (buffer_state_bad(state))
+    if (buffer_state_bad(state, config))
         return result_error(result_error_system_error);
 
     /* Keep the tests short by skipping directories, since behavior shouldn't
@@ -132,10 +142,10 @@ static result_t compress_cctx_compress(
     config_t const* config) {
     buffer_state_t* state = container_of(base, buffer_state_t, base);
 
-    if (buffer_state_bad(state))
+    if (buffer_state_bad(state, config))
         return result_error(result_error_system_error);
 
-    if (config->use_dictionary || config->no_pledged_src_size)
+    if (config->no_pledged_src_size)
         return result_error(result_error_skip);
 
     if (base->data->type != data_type_dir)
@@ -146,8 +156,9 @@ static result_t compress_cctx_compress(
         return result_error(result_error_skip);
 
     ZSTD_CCtx* cctx = ZSTD_createCCtx();
-    if (cctx == NULL) {
-        fprintf(stderr, "ZSTD_createCCtx() failed\n");
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
+    if (cctx == NULL || dctx == NULL) {
+        fprintf(stderr, "context creation failed\n");
         return result_error(result_error_system_error);
     }
 
@@ -156,23 +167,45 @@ static result_t compress_cctx_compress(
     for (size_t i = 0; i < state->inputs.size; ++i) {
         data_buffer_t const input = state->inputs.buffers[i];
 
-        state->compressed.size = ZSTD_compressCCtx(
-            cctx,
-            state->compressed.data,
-            state->compressed.capacity,
-            input.data,
-            input.size,
-            level);
+        if (config->use_dictionary)
+            state->compressed.size = ZSTD_compress_usingDict(
+                cctx,
+                state->compressed.data,
+                state->compressed.capacity,
+                input.data,
+                input.size,
+                state->dictionary.data,
+                state->dictionary.size,
+                level);
+        else
+            state->compressed.size = ZSTD_compressCCtx(
+                cctx,
+                state->compressed.data,
+                state->compressed.capacity,
+                input.data,
+                input.size,
+                level);
         if (ZSTD_isError(state->compressed.size)) {
             result = result_error(result_error_compression_error);
             goto out;
         }
 
-        state->decompressed.size = ZSTD_decompress(
-            state->decompressed.data,
-            state->decompressed.capacity,
-            state->compressed.data,
-            state->compressed.size);
+        if (config->use_dictionary)
+            state->decompressed.size = ZSTD_decompress_usingDict(
+                dctx,
+                state->decompressed.data,
+                state->decompressed.capacity,
+                state->compressed.data,
+                state->compressed.size,
+                state->dictionary.data,
+                state->dictionary.size);
+        else
+            state->decompressed.size = ZSTD_decompressDCtx(
+                dctx,
+                state->decompressed.data,
+                state->decompressed.capacity,
+                state->compressed.data,
+                state->compressed.size);
         if (ZSTD_isError(state->decompressed.size)) {
             result = result_error(result_error_decompression_error);
             goto out;
@@ -188,6 +221,7 @@ static result_t compress_cctx_compress(
     result = result_data(data);
 out:
     ZSTD_freeCCtx(cctx);
+    ZSTD_freeDCtx(dctx);
     return result;
 }
 
@@ -204,9 +238,7 @@ static void method_state_destroy(method_state_t* state) {
     free(state);
 }
 
-static result_t cli_compress(
-    method_state_t* state,
-    config_t const* config) {
+static result_t cli_compress(method_state_t* state, config_t const* config) {
     if (config->cli_args == NULL)
         return result_error(result_error_skip);
 
@@ -257,20 +289,249 @@ static result_t cli_compress(
     return result_data(data);
 }
 
+static int advanced_config(
+    ZSTD_CCtx* cctx,
+    buffer_state_t* state,
+    config_t const* config) {
+    for (size_t p = 0; p < config->param_values.size; ++p) {
+        param_value_t const pv = config->param_values.data[p];
+        if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, pv.param, pv.value))) {
+            return 1;
+        }
+    }
+    if (config->use_dictionary) {
+        if (ZSTD_isError(ZSTD_CCtx_loadDictionary(
+                cctx, state->dictionary.data, state->dictionary.size))) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+static result_t advanced_one_pass_compress_output_adjustment(
+    method_state_t* base,
+    config_t const* config,
+    size_t const subtract) {
+    buffer_state_t* state = container_of(base, buffer_state_t, base);
+
+    if (buffer_state_bad(state, config))
+        return result_error(result_error_system_error);
+
+    ZSTD_CCtx* cctx = ZSTD_createCCtx();
+    result_t result;
+
+    if (!cctx || advanced_config(cctx, state, config)) {
+        result = result_error(result_error_compression_error);
+        goto out;
+    }
+
+    result_data_t data = {.total_size = 0};
+    for (size_t i = 0; i < state->inputs.size; ++i) {
+        data_buffer_t const input = state->inputs.buffers[i];
+
+        if (!config->no_pledged_src_size) {
+            if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) {
+                result = result_error(result_error_compression_error);
+                goto out;
+            }
+        }
+        size_t const size = ZSTD_compress2(
+            cctx,
+            state->compressed.data,
+            ZSTD_compressBound(input.size) - subtract,
+            input.data,
+            input.size);
+        if (ZSTD_isError(size)) {
+            result = result_error(result_error_compression_error);
+            goto out;
+        }
+        data.total_size += size;
+    }
+
+    result = result_data(data);
+out:
+    ZSTD_freeCCtx(cctx);
+    return result;
+}
+
+static result_t advanced_one_pass_compress(
+    method_state_t* base,
+    config_t const* config) {
+  return advanced_one_pass_compress_output_adjustment(base, config, 0);
+}
+
+static result_t advanced_one_pass_compress_small_output(
+    method_state_t* base,
+    config_t const* config) {
+  return advanced_one_pass_compress_output_adjustment(base, config, 1);
+}
+
+static result_t advanced_streaming_compress(
+    method_state_t* base,
+    config_t const* config) {
+    buffer_state_t* state = container_of(base, buffer_state_t, base);
+
+    if (buffer_state_bad(state, config))
+        return result_error(result_error_system_error);
+
+    ZSTD_CCtx* cctx = ZSTD_createCCtx();
+    result_t result;
+
+    if (!cctx || advanced_config(cctx, state, config)) {
+        result = result_error(result_error_compression_error);
+        goto out;
+    }
+
+    result_data_t data = {.total_size = 0};
+    for (size_t i = 0; i < state->inputs.size; ++i) {
+        data_buffer_t input = state->inputs.buffers[i];
+
+        if (!config->no_pledged_src_size) {
+            if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) {
+                result = result_error(result_error_compression_error);
+                goto out;
+            }
+        }
+
+        while (input.size > 0) {
+            ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
+            input.data += in.size;
+            input.size -= in.size;
+            ZSTD_EndDirective const op =
+                input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
+            size_t ret = 0;
+            while (in.pos < in.size || (op == ZSTD_e_end && ret != 0)) {
+                ZSTD_outBuffer out = {state->compressed.data,
+                                      MIN(state->compressed.capacity, 1024)};
+                ret = ZSTD_compressStream2(cctx, &out, &in, op);
+                if (ZSTD_isError(ret)) {
+                    result = result_error(result_error_compression_error);
+                    goto out;
+                }
+                data.total_size += out.pos;
+            }
+        }
+    }
+
+    result = result_data(data);
+out:
+    ZSTD_freeCCtx(cctx);
+    return result;
+}
+
+static result_t old_streaming_compress(
+    method_state_t* base,
+    config_t const* config) {
+    buffer_state_t* state = container_of(base, buffer_state_t, base);
+
+    if (buffer_state_bad(state, config))
+        return result_error(result_error_system_error);
+
+    int const level = config_get_level(config);
+    if (level == CONFIG_NO_LEVEL)
+        return result_error(result_error_skip);
+
+    ZSTD_CStream* zcs = ZSTD_createCStream();
+    result_t result;
+    if (zcs == NULL) {
+        result = result_error(result_error_compression_error);
+        goto out;
+    }
+    size_t zret;
+    if (config->use_dictionary) {
+        zret = ZSTD_initCStream_usingDict(
+            zcs, state->dictionary.data, state->dictionary.size, level);
+    } else {
+        zret = ZSTD_initCStream(zcs, level);
+    }
+    if (ZSTD_isError(zret)) {
+        result = result_error(result_error_compression_error);
+        goto out;
+    }
+
+    result_data_t data = {.total_size = 0};
+    for (size_t i = 0; i < state->inputs.size; ++i) {
+        data_buffer_t input = state->inputs.buffers[i];
+        zret = ZSTD_resetCStream(
+            zcs,
+            config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN
+                                        : input.size);
+        if (ZSTD_isError(zret)) {
+            result = result_error(result_error_compression_error);
+            goto out;
+        }
+
+        while (input.size > 0) {
+            ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
+            input.data += in.size;
+            input.size -= in.size;
+            ZSTD_EndDirective const op =
+                input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
+            zret = 0;
+            while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) {
+                ZSTD_outBuffer out = {state->compressed.data,
+                                      MIN(state->compressed.capacity, 1024)};
+                if (op == ZSTD_e_continue || in.pos < in.size)
+                    zret = ZSTD_compressStream(zcs, &out, &in);
+                else
+                    zret = ZSTD_endStream(zcs, &out);
+                if (ZSTD_isError(zret)) {
+                    result = result_error(result_error_compression_error);
+                    goto out;
+                }
+                data.total_size += out.pos;
+            }
+        }
+    }
+
+    result = result_data(data);
+out:
+    ZSTD_freeCStream(zcs);
+    return result;
+}
+
 method_t const simple = {
-    .name = "ZSTD_compress",
+    .name = "compress simple",
     .create = buffer_state_create,
     .compress = simple_compress,
     .destroy = buffer_state_destroy,
 };
 
 method_t const compress_cctx = {
-    .name = "ZSTD_compressCCtx",
+    .name = "compress cctx",
     .create = buffer_state_create,
     .compress = compress_cctx_compress,
     .destroy = buffer_state_destroy,
 };
 
+method_t const advanced_one_pass = {
+    .name = "advanced one pass",
+    .create = buffer_state_create,
+    .compress = advanced_one_pass_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const advanced_one_pass_small_out = {
+    .name = "advanced one pass small out",
+    .create = buffer_state_create,
+    .compress = advanced_one_pass_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const advanced_streaming = {
+    .name = "advanced streaming",
+    .create = buffer_state_create,
+    .compress = advanced_streaming_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const old_streaming = {
+    .name = "old streaming",
+    .create = buffer_state_create,
+    .compress = old_streaming_compress,
+    .destroy = buffer_state_destroy,
+};
+
 method_t const cli = {
     .name = "zstdcli",
     .create = method_state_create,
@@ -282,6 +543,10 @@ static method_t const* g_methods[] = {
     &simple,
     &compress_cctx,
     &cli,
+    &advanced_one_pass,
+    &advanced_one_pass_small_out,
+    &advanced_streaming,
+    &old_streaming,
     NULL,
 };
 
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index d9f1873ce..3e9d791a5 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -1,101 +1,350 @@
-Data,               Config,             Method,             Total compressed size
-This line is intentionally added to see how the nightly job reports failures
-silesia.tar,        level -5,           ZSTD_compress,      7160438
-silesia.tar,        level -3,           ZSTD_compress,      6789024
-silesia.tar,        level -1,           ZSTD_compress,      6195462
-silesia.tar,        level 0,            ZSTD_compress,      4875071
-silesia.tar,        level 1,            ZSTD_compress,      5339697
-silesia.tar,        level 3,            ZSTD_compress,      4875071
-silesia.tar,        level 4,            ZSTD_compress,      4813104
-silesia.tar,        level 5,            ZSTD_compress,      4726961
-silesia.tar,        level 6,            ZSTD_compress,      4654401
-silesia.tar,        level 7,            ZSTD_compress,      4591933
-silesia.tar,        level 9,            ZSTD_compress,      4554098
-silesia.tar,        level 13,           ZSTD_compress,      4503496
-silesia.tar,        level 16,           ZSTD_compress,      4387233
-silesia.tar,        level 19,           ZSTD_compress,      4283123
-silesia,            level -5,           ZSTD_compressCCtx,  7152294
-silesia,            level -3,           ZSTD_compressCCtx,  6789969
-silesia,            level -1,           ZSTD_compressCCtx,  6191548
-silesia,            level 0,            ZSTD_compressCCtx,  4862377
-silesia,            level 1,            ZSTD_compressCCtx,  5318036
-silesia,            level 3,            ZSTD_compressCCtx,  4862377
-silesia,            level 4,            ZSTD_compressCCtx,  4800629
-silesia,            level 5,            ZSTD_compressCCtx,  4715005
-silesia,            level 6,            ZSTD_compressCCtx,  4644055
-silesia,            level 7,            ZSTD_compressCCtx,  4581559
-silesia,            level 9,            ZSTD_compressCCtx,  4543862
-silesia,            level 13,           ZSTD_compressCCtx,  4493931
-silesia,            level 16,           ZSTD_compressCCtx,  4381885
-silesia,            level 19,           ZSTD_compressCCtx,  4296899
-github,             level -5,           ZSTD_compressCCtx,  232744
-github,             level -3,           ZSTD_compressCCtx,  220611
-github,             level -1,           ZSTD_compressCCtx,  176575
-github,             level 0,            ZSTD_compressCCtx,  136397
-github,             level 1,            ZSTD_compressCCtx,  143457
-github,             level 3,            ZSTD_compressCCtx,  136397
-github,             level 4,            ZSTD_compressCCtx,  136144
-github,             level 5,            ZSTD_compressCCtx,  135106
-github,             level 6,            ZSTD_compressCCtx,  135108
-github,             level 7,            ZSTD_compressCCtx,  135108
-github,             level 9,            ZSTD_compressCCtx,  135108
-github,             level 13,           ZSTD_compressCCtx,  133741
-github,             level 16,           ZSTD_compressCCtx,  133741
-github,             level 19,           ZSTD_compressCCtx,  133717
-silesia,            level -5,           zstdcli,            7152342
-silesia,            level -3,           zstdcli,            6790021
-silesia,            level -1,           zstdcli,            6191597
-silesia,            level 0,            zstdcli,            4862425
-silesia,            level 1,            zstdcli,            5318084
-silesia,            level 3,            zstdcli,            4862425
-silesia,            level 4,            zstdcli,            4800677
-silesia,            level 5,            zstdcli,            4715053
-silesia,            level 6,            zstdcli,            4644103
-silesia,            level 7,            zstdcli,            4581607
-silesia,            level 9,            zstdcli,            4543910
-silesia,            level 13,           zstdcli,            4493979
-silesia,            level 16,           zstdcli,            4381933
-silesia,            level 19,           zstdcli,            4296947
-silesia.tar,        level -5,           zstdcli,            7159586
-silesia.tar,        level -3,           zstdcli,            6791018
-silesia.tar,        level -1,           zstdcli,            6196283
-silesia.tar,        level 0,            zstdcli,            4876730
-silesia.tar,        level 1,            zstdcli,            5340312
-silesia.tar,        level 3,            zstdcli,            4876730
-silesia.tar,        level 4,            zstdcli,            4817723
-silesia.tar,        level 5,            zstdcli,            4730389
-silesia.tar,        level 6,            zstdcli,            4655708
-silesia.tar,        level 7,            zstdcli,            4593407
-silesia.tar,        level 9,            zstdcli,            4556135
-silesia.tar,        level 13,           zstdcli,            4503500
-silesia.tar,        level 16,           zstdcli,            4387237
-silesia.tar,        level 19,           zstdcli,            4283127
-silesia.tar,        no source size,     zstdcli,            4876726
-github,             level -5,           zstdcli,            234744
-github,             level -5 with dict, zstdcli,            47528
-github,             level -3,           zstdcli,            222611
-github,             level -3 with dict, zstdcli,            46394
-github,             level -1,           zstdcli,            178575
-github,             level -1 with dict, zstdcli,            43401
-github,             level 0,            zstdcli,            138397
-github,             level 0 with dict,  zstdcli,            40316
-github,             level 1,            zstdcli,            145457
-github,             level 1 with dict,  zstdcli,            43242
-github,             level 3,            zstdcli,            138397
-github,             level 3 with dict,  zstdcli,            40316
-github,             level 4,            zstdcli,            138144
-github,             level 4 with dict,  zstdcli,            40292
-github,             level 5,            zstdcli,            137106
-github,             level 5 with dict,  zstdcli,            40938
-github,             level 6,            zstdcli,            137108
-github,             level 6 with dict,  zstdcli,            40632
-github,             level 7,            zstdcli,            137108
-github,             level 7 with dict,  zstdcli,            40766
-github,             level 9,            zstdcli,            137108
-github,             level 9 with dict,  zstdcli,            41326
-github,             level 13,           zstdcli,            135741
-github,             level 13 with dict, zstdcli,            41670
-github,             level 16,           zstdcli,            135741
-github,             level 16 with dict, zstdcli,            39940
-github,             level 19,           zstdcli,            135717
-github,             level 19 with dict, zstdcli,            39576
+Data,                        Config,                      Method,                      Total compressed size
+silesia.tar,                 level -5,                    compress simple,             7160438
+silesia.tar,                 level -3,                    compress simple,             6789024
+silesia.tar,                 level -1,                    compress simple,             6195462
+silesia.tar,                 level 0,                     compress simple,             4875071
+silesia.tar,                 level 1,                     compress simple,             5339697
+silesia.tar,                 level 3,                     compress simple,             4875071
+silesia.tar,                 level 4,                     compress simple,             4813104
+silesia.tar,                 level 5,                     compress simple,             4726961
+silesia.tar,                 level 6,                     compress simple,             4654401
+silesia.tar,                 level 7,                     compress simple,             4591933
+silesia.tar,                 level 9,                     compress simple,             4554098
+silesia.tar,                 level 13,                    compress simple,             4503496
+silesia.tar,                 level 16,                    compress simple,             4387233
+silesia.tar,                 level 19,                    compress simple,             4283123
+silesia,                     level -5,                    compress cctx,               7152294
+silesia,                     level -3,                    compress cctx,               6789969
+silesia,                     level -1,                    compress cctx,               6191548
+silesia,                     level 0,                     compress cctx,               4862377
+silesia,                     level 1,                     compress cctx,               5318036
+silesia,                     level 3,                     compress cctx,               4862377
+silesia,                     level 4,                     compress cctx,               4800629
+silesia,                     level 5,                     compress cctx,               4715005
+silesia,                     level 6,                     compress cctx,               4644055
+silesia,                     level 7,                     compress cctx,               4581559
+silesia,                     level 9,                     compress cctx,               4543862
+silesia,                     level 13,                    compress cctx,               4493931
+silesia,                     level 16,                    compress cctx,               4381885
+silesia,                     level 19,                    compress cctx,               4296899
+github,                      level -5,                    compress cctx,               232744
+github,                      level -5 with dict,          compress cctx,               45704
+github,                      level -3,                    compress cctx,               220611
+github,                      level -3 with dict,          compress cctx,               44510
+github,                      level -1,                    compress cctx,               176575
+github,                      level -1 with dict,          compress cctx,               41586
+github,                      level 0,                     compress cctx,               136397
+github,                      level 0 with dict,           compress cctx,               38700
+github,                      level 1,                     compress cctx,               143457
+github,                      level 1 with dict,           compress cctx,               41538
+github,                      level 3,                     compress cctx,               136397
+github,                      level 3 with dict,           compress cctx,               38700
+github,                      level 4,                     compress cctx,               136144
+github,                      level 4 with dict,           compress cctx,               38639
+github,                      level 5,                     compress cctx,               135106
+github,                      level 5 with dict,           compress cctx,               38934
+github,                      level 6,                     compress cctx,               135108
+github,                      level 6 with dict,           compress cctx,               38628
+github,                      level 7,                     compress cctx,               135108
+github,                      level 7 with dict,           compress cctx,               38741
+github,                      level 9,                     compress cctx,               135108
+github,                      level 9 with dict,           compress cctx,               39335
+github,                      level 13,                    compress cctx,               133741
+github,                      level 13 with dict,          compress cctx,               39670
+github,                      level 16,                    compress cctx,               133741
+github,                      level 16 with dict,          compress cctx,               37928
+github,                      level 19,                    compress cctx,               133717
+github,                      level 19 with dict,          compress cctx,               37567
+silesia,                     level -5,                    zstdcli,                     7152342
+silesia,                     level -3,                    zstdcli,                     6790021
+silesia,                     level -1,                    zstdcli,                     6191597
+silesia,                     level 0,                     zstdcli,                     4862425
+silesia,                     level 1,                     zstdcli,                     5318084
+silesia,                     level 3,                     zstdcli,                     4862425
+silesia,                     level 4,                     zstdcli,                     4800677
+silesia,                     level 5,                     zstdcli,                     4715053
+silesia,                     level 6,                     zstdcli,                     4644103
+silesia,                     level 7,                     zstdcli,                     4581607
+silesia,                     level 9,                     zstdcli,                     4543910
+silesia,                     level 13,                    zstdcli,                     4493979
+silesia,                     level 16,                    zstdcli,                     4381933
+silesia,                     level 19,                    zstdcli,                     4296947
+silesia.tar,                 level -5,                    zstdcli,                     7159586
+silesia.tar,                 level -3,                    zstdcli,                     6791018
+silesia.tar,                 level -1,                    zstdcli,                     6196283
+silesia.tar,                 level 0,                     zstdcli,                     4876730
+silesia.tar,                 level 1,                     zstdcli,                     5340312
+silesia.tar,                 level 3,                     zstdcli,                     4876730
+silesia.tar,                 level 4,                     zstdcli,                     4817723
+silesia.tar,                 level 5,                     zstdcli,                     4730389
+silesia.tar,                 level 6,                     zstdcli,                     4655708
+silesia.tar,                 level 7,                     zstdcli,                     4593407
+silesia.tar,                 level 9,                     zstdcli,                     4556135
+silesia.tar,                 level 13,                    zstdcli,                     4503500
+silesia.tar,                 level 16,                    zstdcli,                     4387237
+silesia.tar,                 level 19,                    zstdcli,                     4283127
+silesia.tar,                 no source size,              zstdcli,                     4876726
+github,                      level -5,                    zstdcli,                     234744
+github,                      level -5 with dict,          zstdcli,                     47528
+github,                      level -3,                    zstdcli,                     222611
+github,                      level -3 with dict,          zstdcli,                     46394
+github,                      level -1,                    zstdcli,                     178575
+github,                      level -1 with dict,          zstdcli,                     43401
+github,                      level 0,                     zstdcli,                     138397
+github,                      level 0 with dict,           zstdcli,                     40316
+github,                      level 1,                     zstdcli,                     145457
+github,                      level 1 with dict,           zstdcli,                     43242
+github,                      level 3,                     zstdcli,                     138397
+github,                      level 3 with dict,           zstdcli,                     40316
+github,                      level 4,                     zstdcli,                     138144
+github,                      level 4 with dict,           zstdcli,                     40292
+github,                      level 5,                     zstdcli,                     137106
+github,                      level 5 with dict,           zstdcli,                     40938
+github,                      level 6,                     zstdcli,                     137108
+github,                      level 6 with dict,           zstdcli,                     40632
+github,                      level 7,                     zstdcli,                     137108
+github,                      level 7 with dict,           zstdcli,                     40766
+github,                      level 9,                     zstdcli,                     137108
+github,                      level 9 with dict,           zstdcli,                     41326
+github,                      level 13,                    zstdcli,                     135741
+github,                      level 13 with dict,          zstdcli,                     41670
+github,                      level 16,                    zstdcli,                     135741
+github,                      level 16 with dict,          zstdcli,                     39940
+github,                      level 19,                    zstdcli,                     135717
+github,                      level 19 with dict,          zstdcli,                     39576
+silesia,                     level -5,                    advanced one pass,           7152294
+silesia,                     level -3,                    advanced one pass,           6789969
+silesia,                     level -1,                    advanced one pass,           6191548
+silesia,                     level 0,                     advanced one pass,           4862377
+silesia,                     level 1,                     advanced one pass,           5318036
+silesia,                     level 3,                     advanced one pass,           4862377
+silesia,                     level 4,                     advanced one pass,           4800629
+silesia,                     level 5,                     advanced one pass,           4715005
+silesia,                     level 6,                     advanced one pass,           4644055
+silesia,                     level 7,                     advanced one pass,           4581559
+silesia,                     level 9,                     advanced one pass,           4543862
+silesia,                     level 13,                    advanced one pass,           4493931
+silesia,                     level 16,                    advanced one pass,           4381885
+silesia,                     level 19,                    advanced one pass,           4296899
+silesia,                     no source size,              advanced one pass,           4862377
+silesia.tar,                 level -5,                    advanced one pass,           7160438
+silesia.tar,                 level -3,                    advanced one pass,           6789024
+silesia.tar,                 level -1,                    advanced one pass,           6195462
+silesia.tar,                 level 0,                     advanced one pass,           4875071
+silesia.tar,                 level 1,                     advanced one pass,           5339697
+silesia.tar,                 level 3,                     advanced one pass,           4875071
+silesia.tar,                 level 4,                     advanced one pass,           4813104
+silesia.tar,                 level 5,                     advanced one pass,           4726961
+silesia.tar,                 level 6,                     advanced one pass,           4654401
+silesia.tar,                 level 7,                     advanced one pass,           4591933
+silesia.tar,                 level 9,                     advanced one pass,           4554098
+silesia.tar,                 level 13,                    advanced one pass,           4503496
+silesia.tar,                 level 16,                    advanced one pass,           4387233
+silesia.tar,                 level 19,                    advanced one pass,           4283123
+silesia.tar,                 no source size,              advanced one pass,           4875071
+github,                      level -5,                    advanced one pass,           232744
+github,                      level -5 with dict,          advanced one pass,           45528
+github,                      level -3,                    advanced one pass,           220611
+github,                      level -3 with dict,          advanced one pass,           44394
+github,                      level -1,                    advanced one pass,           176575
+github,                      level -1 with dict,          advanced one pass,           41401
+github,                      level 0,                     advanced one pass,           136397
+github,                      level 0 with dict,           advanced one pass,           38316
+github,                      level 1,                     advanced one pass,           143457
+github,                      level 1 with dict,           advanced one pass,           41242
+github,                      level 3,                     advanced one pass,           136397
+github,                      level 3 with dict,           advanced one pass,           38316
+github,                      level 4,                     advanced one pass,           136144
+github,                      level 4 with dict,           advanced one pass,           38292
+github,                      level 5,                     advanced one pass,           135106
+github,                      level 5 with dict,           advanced one pass,           38938
+github,                      level 6,                     advanced one pass,           135108
+github,                      level 6 with dict,           advanced one pass,           38632
+github,                      level 7,                     advanced one pass,           135108
+github,                      level 7 with dict,           advanced one pass,           38766
+github,                      level 9,                     advanced one pass,           135108
+github,                      level 9 with dict,           advanced one pass,           39326
+github,                      level 13,                    advanced one pass,           133741
+github,                      level 13 with dict,          advanced one pass,           39670
+github,                      level 16,                    advanced one pass,           133741
+github,                      level 16 with dict,          advanced one pass,           37940
+github,                      level 19,                    advanced one pass,           133717
+github,                      level 19 with dict,          advanced one pass,           37576
+github,                      no source size,              advanced one pass,           136397
+silesia,                     level -5,                    advanced one pass small out, 7152294
+silesia,                     level -3,                    advanced one pass small out, 6789969
+silesia,                     level -1,                    advanced one pass small out, 6191548
+silesia,                     level 0,                     advanced one pass small out, 4862377
+silesia,                     level 1,                     advanced one pass small out, 5318036
+silesia,                     level 3,                     advanced one pass small out, 4862377
+silesia,                     level 4,                     advanced one pass small out, 4800629
+silesia,                     level 5,                     advanced one pass small out, 4715005
+silesia,                     level 6,                     advanced one pass small out, 4644055
+silesia,                     level 7,                     advanced one pass small out, 4581559
+silesia,                     level 9,                     advanced one pass small out, 4543862
+silesia,                     level 13,                    advanced one pass small out, 4493931
+silesia,                     level 16,                    advanced one pass small out, 4381885
+silesia,                     level 19,                    advanced one pass small out, 4296899
+silesia,                     no source size,              advanced one pass small out, 4862377
+silesia.tar,                 level -5,                    advanced one pass small out, 7160438
+silesia.tar,                 level -3,                    advanced one pass small out, 6789024
+silesia.tar,                 level -1,                    advanced one pass small out, 6195462
+silesia.tar,                 level 0,                     advanced one pass small out, 4875071
+silesia.tar,                 level 1,                     advanced one pass small out, 5339697
+silesia.tar,                 level 3,                     advanced one pass small out, 4875071
+silesia.tar,                 level 4,                     advanced one pass small out, 4813104
+silesia.tar,                 level 5,                     advanced one pass small out, 4726961
+silesia.tar,                 level 6,                     advanced one pass small out, 4654401
+silesia.tar,                 level 7,                     advanced one pass small out, 4591933
+silesia.tar,                 level 9,                     advanced one pass small out, 4554098
+silesia.tar,                 level 13,                    advanced one pass small out, 4503496
+silesia.tar,                 level 16,                    advanced one pass small out, 4387233
+silesia.tar,                 level 19,                    advanced one pass small out, 4283123
+silesia.tar,                 no source size,              advanced one pass small out, 4875071
+github,                      level -5,                    advanced one pass small out, 232744
+github,                      level -5 with dict,          advanced one pass small out, 45528
+github,                      level -3,                    advanced one pass small out, 220611
+github,                      level -3 with dict,          advanced one pass small out, 44394
+github,                      level -1,                    advanced one pass small out, 176575
+github,                      level -1 with dict,          advanced one pass small out, 41401
+github,                      level 0,                     advanced one pass small out, 136397
+github,                      level 0 with dict,           advanced one pass small out, 38316
+github,                      level 1,                     advanced one pass small out, 143457
+github,                      level 1 with dict,           advanced one pass small out, 41242
+github,                      level 3,                     advanced one pass small out, 136397
+github,                      level 3 with dict,           advanced one pass small out, 38316
+github,                      level 4,                     advanced one pass small out, 136144
+github,                      level 4 with dict,           advanced one pass small out, 38292
+github,                      level 5,                     advanced one pass small out, 135106
+github,                      level 5 with dict,           advanced one pass small out, 38938
+github,                      level 6,                     advanced one pass small out, 135108
+github,                      level 6 with dict,           advanced one pass small out, 38632
+github,                      level 7,                     advanced one pass small out, 135108
+github,                      level 7 with dict,           advanced one pass small out, 38766
+github,                      level 9,                     advanced one pass small out, 135108
+github,                      level 9 with dict,           advanced one pass small out, 39326
+github,                      level 13,                    advanced one pass small out, 133741
+github,                      level 13 with dict,          advanced one pass small out, 39670
+github,                      level 16,                    advanced one pass small out, 133741
+github,                      level 16 with dict,          advanced one pass small out, 37940
+github,                      level 19,                    advanced one pass small out, 133717
+github,                      level 19 with dict,          advanced one pass small out, 37576
+github,                      no source size,              advanced one pass small out, 136397
+silesia,                     level -5,                    advanced streaming,          7152294
+silesia,                     level -3,                    advanced streaming,          6789973
+silesia,                     level -1,                    advanced streaming,          6191549
+silesia,                     level 0,                     advanced streaming,          4862377
+silesia,                     level 1,                     advanced streaming,          5318036
+silesia,                     level 3,                     advanced streaming,          4862377
+silesia,                     level 4,                     advanced streaming,          4800629
+silesia,                     level 5,                     advanced streaming,          4715005
+silesia,                     level 6,                     advanced streaming,          4644055
+silesia,                     level 7,                     advanced streaming,          4581559
+silesia,                     level 9,                     advanced streaming,          4543862
+silesia,                     level 13,                    advanced streaming,          4493931
+silesia,                     level 16,                    advanced streaming,          4381885
+silesia,                     level 19,                    advanced streaming,          4296899
+silesia,                     no source size,              advanced streaming,          4862341
+silesia.tar,                 level -5,                    advanced streaming,          7160440
+silesia.tar,                 level -3,                    advanced streaming,          6789026
+silesia.tar,                 level -1,                    advanced streaming,          6195465
+silesia.tar,                 level 0,                     advanced streaming,          4875071
+silesia.tar,                 level 1,                     advanced streaming,          5339701
+silesia.tar,                 level 3,                     advanced streaming,          4875071
+silesia.tar,                 level 4,                     advanced streaming,          4813104
+silesia.tar,                 level 5,                     advanced streaming,          4726977
+silesia.tar,                 level 6,                     advanced streaming,          4654404
+silesia.tar,                 level 7,                     advanced streaming,          4591934
+silesia.tar,                 level 9,                     advanced streaming,          4554105
+silesia.tar,                 level 13,                    advanced streaming,          4503496
+silesia.tar,                 level 16,                    advanced streaming,          4387233
+silesia.tar,                 level 19,                    advanced streaming,          4283123
+silesia.tar,                 no source size,              advanced streaming,          4875067
+github,                      level -5,                    advanced streaming,          232744
+github,                      level -5 with dict,          advanced streaming,          45528
+github,                      level -3,                    advanced streaming,          220611
+github,                      level -3 with dict,          advanced streaming,          44394
+github,                      level -1,                    advanced streaming,          176575
+github,                      level -1 with dict,          advanced streaming,          41401
+github,                      level 0,                     advanced streaming,          136397
+github,                      level 0 with dict,           advanced streaming,          38316
+github,                      level 1,                     advanced streaming,          143457
+github,                      level 1 with dict,           advanced streaming,          41242
+github,                      level 3,                     advanced streaming,          136397
+github,                      level 3 with dict,           advanced streaming,          38316
+github,                      level 4,                     advanced streaming,          136144
+github,                      level 4 with dict,           advanced streaming,          38292
+github,                      level 5,                     advanced streaming,          135106
+github,                      level 5 with dict,           advanced streaming,          38938
+github,                      level 6,                     advanced streaming,          135108
+github,                      level 6 with dict,           advanced streaming,          38632
+github,                      level 7,                     advanced streaming,          135108
+github,                      level 7 with dict,           advanced streaming,          38766
+github,                      level 9,                     advanced streaming,          135108
+github,                      level 9 with dict,           advanced streaming,          39326
+github,                      level 13,                    advanced streaming,          133741
+github,                      level 13 with dict,          advanced streaming,          39670
+github,                      level 16,                    advanced streaming,          133741
+github,                      level 16 with dict,          advanced streaming,          37940
+github,                      level 19,                    advanced streaming,          133717
+github,                      level 19 with dict,          advanced streaming,          37576
+github,                      no source size,              advanced streaming,          136397
+silesia,                     level -5,                    old streaming,               7152294
+silesia,                     level -3,                    old streaming,               6789973
+silesia,                     level -1,                    old streaming,               6191549
+silesia,                     level 0,                     old streaming,               4862377
+silesia,                     level 1,                     old streaming,               5318036
+silesia,                     level 3,                     old streaming,               4862377
+silesia,                     level 4,                     old streaming,               4800629
+silesia,                     level 5,                     old streaming,               4715005
+silesia,                     level 6,                     old streaming,               4644055
+silesia,                     level 7,                     old streaming,               4581559
+silesia,                     level 9,                     old streaming,               4543862
+silesia,                     level 13,                    old streaming,               4493931
+silesia,                     level 16,                    old streaming,               4381885
+silesia,                     level 19,                    old streaming,               4296899
+silesia,                     no source size,              old streaming,               4862341
+silesia.tar,                 level -5,                    old streaming,               7160440
+silesia.tar,                 level -3,                    old streaming,               6789026
+silesia.tar,                 level -1,                    old streaming,               6195465
+silesia.tar,                 level 0,                     old streaming,               4875071
+silesia.tar,                 level 1,                     old streaming,               5339701
+silesia.tar,                 level 3,                     old streaming,               4875071
+silesia.tar,                 level 4,                     old streaming,               4813104
+silesia.tar,                 level 5,                     old streaming,               4726977
+silesia.tar,                 level 6,                     old streaming,               4654404
+silesia.tar,                 level 7,                     old streaming,               4591934
+silesia.tar,                 level 9,                     old streaming,               4554105
+silesia.tar,                 level 13,                    old streaming,               4503496
+silesia.tar,                 level 16,                    old streaming,               4387233
+silesia.tar,                 level 19,                    old streaming,               4283123
+silesia.tar,                 no source size,              old streaming,               4875067
+github,                      level -5,                    old streaming,               232744
+github,                      level -5 with dict,          old streaming,               45528
+github,                      level -3,                    old streaming,               220611
+github,                      level -3 with dict,          old streaming,               44394
+github,                      level -1,                    old streaming,               176575
+github,                      level -1 with dict,          old streaming,               41401
+github,                      level 0,                     old streaming,               136397
+github,                      level 0 with dict,           old streaming,               38316
+github,                      level 1,                     old streaming,               143457
+github,                      level 1 with dict,           old streaming,               41242
+github,                      level 3,                     old streaming,               136397
+github,                      level 3 with dict,           old streaming,               38316
+github,                      level 4,                     old streaming,               136144
+github,                      level 4 with dict,           old streaming,               38292
+github,                      level 5,                     old streaming,               135106
+github,                      level 5 with dict,           old streaming,               38938
+github,                      level 6,                     old streaming,               135108
+github,                      level 6 with dict,           old streaming,               38632
+github,                      level 7,                     old streaming,               135108
+github,                      level 7 with dict,           old streaming,               38766
+github,                      level 9,                     old streaming,               135108
+github,                      level 9 with dict,           old streaming,               39326
+github,                      level 13,                    old streaming,               133741
+github,                      level 13 with dict,          old streaming,               39670
+github,                      level 16,                    old streaming,               133741
+github,                      level 16 with dict,          old streaming,               37940
+github,                      level 19,                    old streaming,               133717
+github,                      level 19 with dict,          old streaming,               37576
+github,                      no source size,              old streaming,               141003

From 8c99e311cf5ca64cc4f37d223e83eb45ef6fff4f Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Tue, 11 Dec 2018 15:57:56 -0800
Subject: [PATCH 62/62] Reset the cctx for documentation/safety

---
 tests/regression/method.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/regression/method.c b/tests/regression/method.c
index 73d8bbf59..a43e91489 100644
--- a/tests/regression/method.c
+++ b/tests/regression/method.c
@@ -293,6 +293,7 @@ static int advanced_config(
     ZSTD_CCtx* cctx,
     buffer_state_t* state,
     config_t const* config) {
+    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
     for (size_t p = 0; p < config->param_values.size; ++p) {
         param_value_t const pv = config->param_values.data[p];
         if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, pv.param, pv.value))) {