Merge branch 'dev' into btultra2

.circleci/config.yml

@@ -15,7 +15,8 @@ references:
         sudo apt-get -y install \
             gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
             libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
-            libc6-dev-ppc64-powerpc-cross zstd gzip coreutils
+            libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
+            libcurl4-openssl-dev
 
 jobs:
   # the first half of the jobs are in this test
@@ -82,6 +83,49 @@ jobs:
             cp $ZSTD_VERSION.tar* $CIRCLE_ARTIFACTS
       - store_artifacts:
           path: /tmp/circleci-artifacts
+  # This step should only be run in a cron job
+  regression-test:
+    docker:
+      - image: circleci/buildpack-deps:bionic
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+    steps:
+      - checkout
+      - *install-dependencies
+      # Restore the cached resources.
+      - restore_cache:
+          # We try our best to bust the cache when the data changes by hashing
+          # data.c. If that doesn't work, simply update the version number here
+          # and below. If we fail to bust the cache, the regression testing will
+          # still work, since it has its own stamp, but will need to redownload
+          # everything.
+          keys:
+            - regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+      - run:
+          name: Regression Test
+          command: |
+            make -C programs zstd
+            make -C tests/regression test
+            mkdir -p $CIRCLE_ARTIFACTS
+            ./tests/regression/test                     \
+                --cache  tests/regression/cache         \
+                --output $CIRCLE_ARTIFACTS/results.csv  \
+                --zstd   programs/zstd
+            echo "NOTE: The new results.csv is uploaded as an artifact to this job"
+            echo "      If this fails, go to the Artifacts pane in CircleCI, "
+            echo "      download /tmp/circleci-artifacts/results.csv, and if they "
+            echo "      are still good, copy it into the repo and commit it."
+            echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
+            diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
+      # Only save the cache on success (default), since if the failure happened
+      # before we stamp the data cache, we will have a bad cache for this key.
+      - save_cache:
+          key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+          paths:
+            - tests/regression/cache
+      - store_artifacts:
+          path: /tmp/circleci-artifacts
+
 
 workflows:
   version: 2
@@ -96,6 +140,13 @@ workflows:
           filters:
             tags:
               only: /.*/
+      # Create a branch called regression and set it to dev to force a
+      # regression test run
+      - regression-test:
+          filters:
+            branches:
+              only:
+                - regression
       # Only run on release tags.
       - publish-github-release:
           requires:
@@ -106,6 +157,20 @@ workflows:
               ignore: /.*/
             tags:
               only: /^v\d+\.\d+\.\d+$/
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+                - dev
+    jobs:
+      # Run daily long regression tests
+      - regression-test
+
+
 
   # Longer tests
     #- make -C tests test-zstd-nolegacy && make clean

.travis.yml

@@ -3,11 +3,13 @@
 language: c
 dist: trusty
 sudo: required
+
 addons:
   apt:
     update: true
 
 matrix:
+  fast_finish: true
   include:
     # Ubuntu 14.04
     - env: Cmd='make test'
@@ -49,6 +51,11 @@ matrix:
     - if: tag =~ ^v[0-9]\.[0-9]
       env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH'
 
+    - dist: xenial
+      env: BUILD_SYSTEM='meson'
+  allow_failures:
+    - env: BUILD_SYSTEM='meson'
+
 git:
   depth: 1
 
@@ -59,9 +66,28 @@ branches:
   - travisTest
 
 script:
-  - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
+  - JOB_NUMBER=$(printf '%s' "${TRAVIS_JOB_NUMBER}" | sed -E 's@[0-9]+\.([0-9]+)@\1@')
-  - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
+  - printf 'JOB_NUMBER=%s TRAVIS_BRANCH=%s TRAVIS_EVENT_TYPE=%s TRAVIS_PULL_REQUEST=%s\n'
+      "${JOB_NUMBER}" "${TRAVIS_BRANCH}" "${TRAVIS_EVENT_TYPE}" "${TRAVIS_PULL_REQUEST}"
+  - if [ "${BUILD_SYSTEM}" = meson ]; then
+      set -x;
+      sudo apt-get install -qq liblz4-dev valgrind tree
+        && curl -o ~/get-pip.py 'https://bootstrap.pypa.io/get-pip.py'
+        && python3 ~/get-pip.py --user
+        && pip3 install --user meson ninja
+        && export CC=clang CXX=clang++
+        && meson --buildtype=debug
+                 -Db_lundef=false
+                 -Dauto_features=enabled
+                 -Dbuild_{programs,tests,contrib}=true
+                 -Ddefault_library=both
+                 build/meson builddir
+        && cd "$_"
+        && DESTDIR=./staging ninja install
+        && tree ./staging;
+      travis_terminate "$?";
+    fi
   - export FUZZERTEST=-T2mn;
     export ZSTREAM_TESTTIME=-T2mn;
     export DECODECORPUS_TESTTIME=-T1mn;
-    sh -c "$Cmd" || travis_terminate 1;
+    sh -c "${Cmd}" || travis_terminate 1;

Makefile

@@ -130,7 +130,12 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD Dr
 HOST_OS = POSIX
 CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
 
-EGREP = egrep --color=never
+HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+EGREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+EGREP_OPTIONS += --color=never
+endif
+EGREP = egrep $(EGREP_OPTIONS)
 
 # Print a two column output of targets and their description. To add a target description, put a
 # comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>".  For example:

README.md

@@ -125,7 +125,7 @@ By default, `CMAKE_BUILD_TYPE` is set to `Release`.
 
 #### Meson
 
-A Meson project is provided within `contrib/meson`.
+A Meson project is provided within `build/meson`.
 
 #### Visual Studio (Windows)
 
@@ -135,6 +135,10 @@ Going into `build` directory, you will find additional possibilities:
 - Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
   which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
 
+#### Buck
+
+You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
+The output binary will be in `buck-out/gen/programs/`.
 
 ### Status
 

build/meson/GetZstdLibraryVersion.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+# #############################################################################
+# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+import re
+import sys
+
+
+def find_version(filepath):
+  version_file_data = None
+  with open(filepath) as fd:
+    version_file_data = fd.read()
+
+  patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+)
+#\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+)
+#\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+)
+"""
+  regex = re.compile(patterns, re.MULTILINE)
+  version_match = regex.search(version_file_data)
+  if version_match:
+    return version_match.groups()
+  raise Exception("Unable to find version string.")
+
+
+def main():
+  import argparse
+  parser = argparse.ArgumentParser(description='Print zstd version from lib/zstd.h')
+  parser.add_argument('file', help='path to lib/zstd.h')
+  args = parser.parse_args()
+  filepath = args.file
+  version_tup = find_version(filepath)
+  print('.'.join(version_tup))
+
+
+if __name__ == '__main__':
+  main()

build/meson/InstallSymlink.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# #############################################################################
+# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+import errno
+import os
+
+
+def mkdir_p(path, dir_mode=0o777):
+  try:
+    os.makedirs(path, mode=dir_mode)
+  except OSError as exc:  # Python >2.5
+    if exc.errno == errno.EEXIST and os.path.isdir(path):
+      pass
+    else:
+      raise
+
+
+def install_symlink(src, dst, install_dir, dst_is_dir=False, dir_mode=0o777):
+  if not os.path.exists(install_dir):
+      mkdir_p(install_dir, dir_mode)
+  if not os.path.isdir(install_dir):
+      raise NotADirectoryError(install_dir)
+
+  new_dst = os.path.join(install_dir, dst)
+  if os.path.islink(new_dst) and os.readlink(new_dst) == src:
+    print('File exists: {!r} -> {!r}'.format(new_dst, src))
+    return
+  print('Installing symlink {!r} -> {!r}'.format(new_dst, src))
+  os.symlink(src, new_dst, dst_is_dir)
+
+
+def main():
+  import argparse
+  parser = argparse.ArgumentParser(description='Install a symlink',
+      usage='InstallSymlink.py [-h] [-d] [-m MODE] src dst install_dir\n\n'
+            'example:\n'
+            '\tInstallSymlink.py dash sh /bin\n'
+            '\tDESTDIR=./staging InstallSymlink.py dash sh /bin')
+  parser.add_argument('src', help='target to link')
+  parser.add_argument('dst', help='link name')
+  parser.add_argument('install_dir', help='installation directory')
+  parser.add_argument('-d', '--isdir',
+      action='store_true',
+      help='dst is a directory')
+  parser.add_argument('-m', '--mode',
+      help='directory mode on creating if not exist',
+      default='0o777')
+  args = parser.parse_args()
+
+  src = args.src
+  dst = args.dst
+  install_dir = args.install_dir
+  dst_is_dir = args.isdir
+  dir_mode = int(args.mode, 8)
+
+  DESTDIR = os.environ.get('DESTDIR')
+  if DESTDIR:
+      install_dir = DESTDIR + install_dir if os.path.isabs(install_dir) \
+               else os.path.join(DESTDIR, install_dir)
+
+  install_symlink(src, dst, install_dir, dst_is_dir, dir_mode)
+
+
+if __name__ == '__main__':
+  main()

build/meson/README.md

@@ -0,0 +1,38 @@
+Meson build system for zstandard
+================================
+
+Meson is a build system designed to optimize programmer productivity.
+It aims to do this by providing simple, out-of-the-box support for
+modern software development tools and practices, such as unit tests,
+coverage reports, Valgrind, CCache and the like.
+
+This Meson build system is provided with no guarantee and maintained
+by Dima Krasner <dima@dimakrasner.com>.
+
+It outputs one `libzstd`, either shared or static, depending on
+`default_library` option.
+
+## How to build
+
+`cd` to this meson directory (`build/meson`)
+
+```sh
+meson --buildtype=release -D with-contrib=true -D with-tests=true -D with-contrib=true builddir
+cd builddir
+ninja             # to build
+ninja install     # to install
+```
+
+You might want to install it in staging directory:
+
+```sh
+DESTDIR=./staging ninja install
+```
+
+To configure build options, use:
+
+```sh
+meson configure
+```
+
+See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html).

build/meson/contrib/gen_html/meson.build

@@ -0,0 +1,29 @@
+# #############################################################################
+# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../../..'
+
+gen_html_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
+  join_paths(zstd_rootdir, 'lib'),
+  join_paths(zstd_rootdir, 'lib/common'),
+  join_paths(zstd_rootdir, 'contrib/gen_html'))
+
+gen_html = executable('gen_html',
+  join_paths(zstd_rootdir, 'contrib/gen_html/gen_html.cpp'),
+  include_directories: gen_html_includes,
+  install: false)
+
+# Update zstd manual
+zstd_manual_html = custom_target('zstd_manual.html',
+  output : 'zstd_manual.html',
+  command : [gen_html,
+    zstd_version,
+    join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h'),
+    '@OUTPUT@'],
+  install : false)

build/meson/contrib/meson.build

@@ -0,0 +1,12 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+subdir('pzstd')
+subdir('gen_html')

build/meson/contrib/pzstd/meson.build

@@ -0,0 +1,24 @@
+# #############################################################################
+# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../../..'
+
+pzstd_includes = include_directories(join_paths(zstd_rootdir, 'programs'),
+  join_paths(zstd_rootdir, 'contrib/pzstd'))
+pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'contrib/pzstd/main.cpp'),
+  join_paths(zstd_rootdir, 'contrib/pzstd/Options.cpp'),
+  join_paths(zstd_rootdir, 'contrib/pzstd/Pzstd.cpp'),
+  join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')]
+pzstd = executable('pzstd',
+  pzstd_sources,
+  cpp_args: [ '-DNDEBUG', '-Wno-shadow', '-pedantic' ],
+  include_directories: pzstd_includes,
+  dependencies: [ libzstd_dep, thread_dep ],
+  install: true)

build/meson/lib/meson.build

@@ -0,0 +1,130 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../..'
+
+libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'),
+  join_paths(zstd_rootdir, 'lib/common'),
+  join_paths(zstd_rootdir, 'lib/compress'),
+  join_paths(zstd_rootdir, 'lib/decompress'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder'),
+  join_paths(zstd_rootdir, 'lib/deprecated'))]
+
+libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
+  join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'),
+  join_paths(zstd_rootdir, 'lib/common/threading.c'),
+  join_paths(zstd_rootdir, 'lib/common/pool.c'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
+  join_paths(zstd_rootdir, 'lib/common/error_private.c'),
+  join_paths(zstd_rootdir, 'lib/common/xxhash.c'),
+  join_paths(zstd_rootdir, 'lib/compress/hist.c'),
+  join_paths(zstd_rootdir, 'lib/compress/fse_compress.c'),
+  join_paths(zstd_rootdir, 'lib/compress/huf_compress.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_compress.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_lazy.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_opt.c'),
+  join_paths(zstd_rootdir, 'lib/compress/zstd_ldm.c'),
+  join_paths(zstd_rootdir, 'lib/decompress/huf_decompress.c'),
+  join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress.c'),
+  join_paths(zstd_rootdir, 'lib/decompress/zstd_decompress_block.c'),
+  join_paths(zstd_rootdir, 'lib/decompress/zstd_ddict.c'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')]
+
+# Explicit define legacy support
+add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level),
+  language: 'c')
+
+if legacy_level == 0
+  message('Legacy support: DISABLED')
+else
+  # See ZSTD_LEGACY_SUPPORT of lib/README.md
+  message('Enable legacy support back to version 0.@0@'.format(legacy_level))
+
+  libzstd_includes += [ include_directories(join_paths(zstd_rootdir, 'lib/legacy')) ]
+  foreach i : [1, 2, 3, 4, 5, 6, 7]
+    if legacy_level <= i
+      libzstd_sources += join_paths(zstd_rootdir, 'lib/legacy/zstd_v0@0@.c'.format(i))
+    endif
+  endforeach
+endif
+
+libzstd_deps = []
+if use_multi_thread
+  message('Enable multi-threading support')
+  add_project_arguments('-DZSTD_MULTITHREAD', language: 'c')
+  libzstd_deps = [ thread_dep ]
+endif
+
+libzstd_c_args = []
+if cc_id == compiler_msvc
+  if default_library_type != 'static'
+    libzstd_sources += [windows_mod.compile_resources(
+      join_paths(zstd_rootdir, 'build/VS2010/libzstd-dll/libzstd-dll.rc'))]
+    libzstd_c_args += ['-DZSTD_DLL_EXPORT=1',
+      '-DZSTD_HEAPMODE=0',
+      '-D_CONSOLE',
+      '-D_CRT_SECURE_NO_WARNINGS']
+  else
+    libzstd_c_args += ['-DZSTD_HEAPMODE=0',
+      '-D_CRT_SECURE_NO_WARNINGS']
+  endif
+endif
+
+mingw_ansi_stdio_flags = []
+if host_machine_os == os_windows and cc_id == compiler_gcc
+  mingw_ansi_stdio_flags = [ '-D__USE_MINGW_ANSI_STDIO' ]
+endif
+libzstd_c_args += mingw_ansi_stdio_flags
+
+libzstd_debug_cflags = []
+if use_debug
+  libzstd_c_args += '-DDEBUGLEVEL=@0@'.format(debug_level)
+  if cc_id == compiler_gcc or cc_id == compiler_clang
+    libzstd_debug_cflags = ['-Wstrict-aliasing=1', '-Wswitch-enum',
+      '-Wdeclaration-after-statement', '-Wstrict-prototypes',
+      '-Wundef', '-Wpointer-arith', '-Wformat-security', '-Wvla',
+      '-Wformat=2', '-Winit-self', '-Wfloat-equal', '-Wwrite-strings',
+      '-Wredundant-decls', '-Wmissing-prototypes', '-Wc++-compat']
+  endif
+endif
+libzstd_c_args += cc.get_supported_arguments(libzstd_debug_cflags)
+
+libzstd = library('zstd',
+  libzstd_sources,
+  include_directories: libzstd_includes,
+  c_args: libzstd_c_args,
+  dependencies: libzstd_deps,
+  install: true,
+  version: zstd_libversion,
+  soversion: '1')
+
+libzstd_dep = declare_dependency(link_with: libzstd,
+  include_directories: libzstd_includes)
+
+pkgconfig.generate(name: 'libzstd',
+  filebase: 'libzstd',
+  libraries: [libzstd],
+  description: 'fast lossless compression algorithm library',
+  version: zstd_libversion,
+  url: 'http://www.zstd.net/')
+
+install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
+  join_paths(zstd_rootdir, 'lib/deprecated/zbuff.h'),
+  join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_errors.h'))

build/meson/meson.build

@@ -0,0 +1,171 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+project('zstd',
+  ['c', 'cpp'],
+  license: ['BSD', 'GPLv2'],
+  default_options : ['c_std=c99',
+    'cpp_std=c++11',
+    'buildtype=release'],
+  version: '1.3.8',
+  meson_version: '>=0.47.0')
+
+cc = meson.get_compiler('c')
+cxx = meson.get_compiler('cpp')
+pkgconfig = import('pkgconfig')
+python3 = import('python').find_installation()
+windows_mod = import('windows')
+
+host_machine_os = host_machine.system()
+os_windows = 'windows'
+os_linux = 'linux'
+os_darwin = 'darwin'
+os_freebsd = 'freebsd'
+os_sun = 'sunos'
+
+cc_id = cc.get_id()
+compiler_gcc = 'gcc'
+compiler_clang = 'clang'
+compiler_msvc = 'msvc'
+
+zstd_version = meson.project_version()
+zstd_libversion = ''
+
+# =============================================================================
+# Project directories
+# =============================================================================
+
+zstd_rootdir = '../..'
+
+# =============================================================================
+# Installation directories
+# =============================================================================
+
+if host_machine_os == os_windows
+  zstd_prefix = '.'
+  zstd_bindir = 'bin'
+  zstd_datadir = 'share'
+  zstd_mandir = join_paths(zstd_datadir, 'man')
+else
+  zstd_prefix = get_option('prefix')
+  zstd_bindir = join_paths(zstd_prefix, get_option('bindir'))
+  zstd_datadir = join_paths(zstd_prefix, get_option('datadir'))
+  zstd_mandir = join_paths(zstd_prefix, get_option('mandir'))
+endif
+
+zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name())
+
+# =============================================================================
+# Project options
+# =============================================================================
+
+# Built-in options
+use_debug = get_option('debug')
+buildtype = get_option('buildtype')
+
+# Custom options
+debug_level = get_option('debug_level')
+legacy_level = get_option('legacy_level')
+use_backtrace = get_option('backtrace')
+use_static_runtime = get_option('static_runtime')
+
+build_programs = get_option('build_programs')
+build_contrib = get_option('build_contrib')
+build_tests = get_option('build_tests')
+
+feature_multi_thread = get_option('multi_thread')
+feature_zlib = get_option('zlib')
+feature_lzma = get_option('lzma')
+feature_lz4 = get_option('lz4')
+
+# =============================================================================
+# Helper scripts for Meson
+# =============================================================================
+
+GetZstdLibraryVersion_py = files('GetZstdLibraryVersion.py')
+
+# =============================================================================
+# Getting project version from zstd.h
+# =============================================================================
+
+zstd_h_file = join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h')
+r = run_command(python3, GetZstdLibraryVersion_py, zstd_h_file)
+if r.returncode() == 0
+  output = r.stdout().strip()
+  if output.version_compare('>@0@'.format(zstd_version))
+    zstd_version = output
+    message('Project version is now: @0@'.format(zstd_version))
+  endif
+endif
+
+if host_machine_os != os_windows
+  zstd_libversion = zstd_version
+endif
+
+# =============================================================================
+# Dependencies
+# =============================================================================
+
+libm_dep = cc.find_library('m', required: build_tests)
+thread_dep = dependency('threads', required: feature_multi_thread)
+use_multi_thread = thread_dep.found()
+# Arguments in dependency should be equivalent to those passed to pkg-config
+zlib_dep = dependency('zlib', required: feature_zlib)
+use_zlib = zlib_dep.found()
+lzma_dep = dependency('liblzma', required: feature_lzma)
+use_lzma = lzma_dep.found()
+lz4_dep = dependency('liblz4', required: feature_lz4)
+use_lz4 = lz4_dep.found()
+
+# =============================================================================
+# Compiler flags
+# =============================================================================
+
+add_project_arguments('-DXXH_NAMESPACE=ZSTD_', language: ['c'])
+
+if [compiler_gcc, compiler_clang].contains(cc_id)
+  common_warning_flags = [ '-Wextra', '-Wundef', '-Wshadow', '-Wcast-align', '-Wcast-qual' ]
+  if cc_id == compiler_clang
+    # Should use Meson's own --werror build option
+    #common_warning_flags += '-Werror'
+    common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
+  endif
+  cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
+  cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags)
+  add_project_arguments(cc_compile_flags, language : 'c')
+  add_project_arguments(cxx_compile_flags, language : 'cpp')
+elif cc_id == compiler_msvc
+  msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ]
+  if use_multi_thread
+    msvc_compile_flags += '/MP'
+  endif
+  if enable_static_runtime
+    msvc_compile_flags += '/MT'
+  endif
+  add_project_arguments(msvc_compile_flags, language: ['c', 'cpp'])
+endif
+
+# =============================================================================
+# Subdirs
+# =============================================================================
+
+subdir('lib')
+
+if build_programs
+  subdir('programs')
+endif
+
+if build_tests
+  subdir('tests')
+endif
+
+if build_contrib
+  subdir('contrib')
+endif

build/meson/meson_options.txt

@@ -0,0 +1,36 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
+
+option('legacy_level', type: 'integer', min: 0, max: 7, value: '5',
+  description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
+option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
+  description: 'Enable run-time debug. See lib/common/debug.h')
+option('backtrace', type: 'boolean', value: false,
+  description: 'Display a stack backtrace when execution generates a runtime exception')
+option('static_runtime', type: 'boolean', value: false,
+  description: 'Link to static run-time libraries on MSVC')
+
+option('build_programs', type: 'boolean', value: true,
+  description: 'Enable programs build')
+option('build_tests', type: 'boolean', value: false,
+  description: 'Enable tests build')
+option('build_contrib', type: 'boolean', value: false,
+  description: 'Enable contrib build')
+
+option('multi_thread', type: 'feature', value: 'enabled',
+  description: 'Enable multi-threading when pthread is detected')
+option('zlib', type: 'feature', value: 'auto',
+  description: 'Enable zlib support')
+option('lzma', type: 'feature', value: 'auto',
+  description: 'Enable lzma support')
+option('lz4', type: 'feature', value: 'auto',
+  description: 'Enable lz4 support')

build/meson/programs/meson.build

@@ -0,0 +1,101 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../..'
+
+zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/fileio.c'),
+  join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
+  join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/dibio.c')]
+
+zstd_c_args = libzstd_debug_cflags
+if use_multi_thread
+  zstd_c_args += [ '-DZSTD_MULTITHREAD' ]
+endif
+
+zstd_deps = [ libzstd_dep ]
+if use_zlib
+  zstd_deps += [ zlib_dep ]
+  zstd_c_args += [ '-DZSTD_GZCOMPRESS', '-DZSTD_GZDECOMPRESS' ]
+endif
+
+if use_lzma
+  zstd_deps += [ lzma_dep ]
+  zstd_c_args += [ '-DZSTD_LZMACOMPRESS', '-DZSTD_LZMADECOMPRESS' ]
+endif
+
+if use_lz4
+  zstd_deps += [ lz4_dep ]
+  zstd_c_args += [ '-DZSTD_LZ4COMPRESS', '-DZSTD_LZ4DECOMPRESS' ]
+endif
+
+export_dynamic_on_windows = false
+# explicit backtrace enable/disable for Linux & Darwin
+if not use_backtrace
+  zstd_c_args += '-DBACKTRACE_ENABLE=0'
+elif use_debug and host_machine_os == os_windows  # MinGW target
+  zstd_c_args += '-DBACKTRACE_ENABLE=1'
+  export_dynamic_on_windows = true
+endif
+
+if cc_id == compiler_msvc
+  if default_library_type != 'static'
+    zstd_programs_sources += [windows_mod.compile_resources(
+      join_paths(zstd_rootdir, 'build/VS2010/zstd/zstd.rc'))]
+  endif
+endif
+
+zstd = executable('zstd',
+  zstd_programs_sources,
+  c_args: zstd_c_args,
+  dependencies: zstd_deps,
+  export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
+  install: true)
+
+zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/fileio.c')]
+
+# Minimal target, with only zstd compression and decompression.
+# No bench. No legacy.
+executable('zstd-frugal',
+  zstd_frugal_sources,
+  dependencies: libzstd_dep,
+  c_args: [ '-DZSTD_NOBENCH', '-DZSTD_NODICT' ],
+  install: true)
+
+install_data(join_paths(zstd_rootdir, 'programs/zstdgrep'),
+  join_paths(zstd_rootdir, 'programs/zstdless'),
+  install_dir: zstd_bindir)
+
+# =============================================================================
+# Programs and manpages installing
+# =============================================================================
+
+install_man(join_paths(zstd_rootdir, 'programs/zstd.1'),
+  join_paths(zstd_rootdir, 'programs/zstdgrep.1'),
+  join_paths(zstd_rootdir, 'programs/zstdless.1'))
+
+InstallSymlink_py = '../InstallSymlink.py'
+zstd_man1_dir = join_paths(zstd_mandir, 'man1')
+man1_EXT = host_machine_os != os_windows ? '.1.gz' : '.1'
+
+foreach f : ['zstdcat', 'unzstd']
+  meson.add_install_script(InstallSymlink_py, 'zstd', f, zstd_bindir)
+  meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, f + man1_EXT, zstd_man1_dir)
+endforeach
+
+if use_multi_thread
+  meson.add_install_script(InstallSymlink_py, 'zstd', 'zstdmt', zstd_bindir)
+  meson.add_install_script(InstallSymlink_py, 'zstd' + man1_EXT, 'zstdmt' + man1_EXT, zstd_man1_dir)
+endif

build/meson/tests/meson.build

@@ -0,0 +1,218 @@
+# #############################################################################
+# Copyright (c) 2018-present     Dima Krasner <dima@dimakrasner.com>
+#                                lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+
+zstd_rootdir = '../../..'
+
+tests_supported_oses = [os_linux, 'gnu/kfreebsd', os_darwin, 'gnu', 'openbsd',
+  os_freebsd, 'netbsd', 'dragonfly', os_sun]
+
+# =============================================================================
+# Test flags
+# =============================================================================
+
+FUZZER_FLAGS = ['--no-big-tests']
+FUZZERTEST = '-T200s'
+ZSTREAM_TESTTIME = '-T90s'
+DECODECORPUS_TESTTIME = '-T30'
+ZSTDRTTEST = ['--test-large-data']
+
+# =============================================================================
+# Executables
+# =============================================================================
+
+test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]
+
+datagen_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'tests/datagencli.c')]
+datagen = executable('datagen',
+  datagen_sources,
+  c_args: [ '-DNDEBUG' ],
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+fullbench_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
+  join_paths(zstd_rootdir, 'tests/fullbench.c')]
+fullbench = executable('fullbench',
+  fullbench_sources,
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+fuzzer_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'tests/fuzzer.c')]
+fuzzer = executable('fuzzer',
+  fuzzer_sources,
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+zbufftest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'tests/zbufftest.c')]
+zbufftest = executable('zbufftest',
+  zbufftest_sources,
+  c_args: ['-Wno-deprecated-declarations'],
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+zstreamtest_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'tests/seqgen.c'),
+  join_paths(zstd_rootdir, 'tests/zstreamtest.c')]
+zstreamtest = executable('zstreamtest',
+  zstreamtest_sources,
+  include_directories: test_includes,
+  dependencies: libzstd_dep,
+  install: false)
+
+paramgrill_sources = [join_paths(zstd_rootdir, 'programs/benchfn.c'),
+  join_paths(zstd_rootdir, 'programs/benchzstd.c'),
+  join_paths(zstd_rootdir, 'programs/datagen.c'),
+  join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'tests/paramgrill.c')]
+paramgrill = executable('paramgrill',
+  paramgrill_sources,
+  include_directories: test_includes,
+  dependencies: [ libzstd_dep, libm_dep ],
+  install: false)
+
+roundTripCrash_sources = [join_paths(zstd_rootdir, 'tests/roundTripCrash.c')]
+roundTripCrash = executable('roundTripCrash',
+  roundTripCrash_sources,
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
+longmatch = executable('longmatch',
+  longmatch_sources,
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')]
+invalidDictionaries = executable('invalidDictionaries',
+  invalidDictionaries_sources,
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')]
+legacy = executable('legacy',
+  legacy_sources,
+  # Use -Dlegacy_level build option to control it
+  #c_args: '-DZSTD_LEGACY_SUPPORT=4',
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+decodecorpus_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'tests/decodecorpus.c')]
+decodecorpus = executable('decodecorpus',
+  decodecorpus_sources,
+  include_directories: test_includes,
+  dependencies: [ libzstd_dep, libm_dep ],
+  install: false)
+
+symbols_sources = [join_paths(zstd_rootdir, 'tests/symbols.c')]
+symbols = executable('symbols',
+  symbols_sources,
+  include_directories: test_includes,
+  c_args: host_machine_os == os_windows ? '-DZSTD_DLL_IMPORT=1' : [],
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+poolTests_sources = [join_paths(zstd_rootdir, 'programs/util.c'),
+  join_paths(zstd_rootdir, 'tests/poolTests.c'),
+  join_paths(zstd_rootdir, 'lib/common/pool.c'),
+  join_paths(zstd_rootdir, 'lib/common/threading.c'),
+  join_paths(zstd_rootdir, 'lib/common/zstd_common.c'),
+  join_paths(zstd_rootdir, 'lib/common/error_private.c')]
+poolTests = executable('poolTests',
+  poolTests_sources,
+  include_directories: test_includes,
+  dependencies: [ libzstd_dep, thread_dep ],
+  install: false)
+
+checkTag_sources = [join_paths(zstd_rootdir, 'tests/checkTag.c')]
+checkTag = executable('checkTag',
+  checkTag_sources,
+  dependencies: [ libzstd_dep ],
+  install: false)
+
+# =============================================================================
+# Tests (Use "meson test --list" to list all tests)
+# =============================================================================
+
+if tests_supported_oses.contains(host_machine_os)
+  valgrind_prog = find_program('valgrind', ['/usr/bin/valgrind'], required: true)
+  valgrindTest_py = files('valgrindTest.py')
+  test('valgrindTest',
+    valgrindTest_py,
+    args: [valgrind_prog.path(), zstd, datagen, fuzzer, fullbench],
+    depends: [zstd, datagen, fuzzer, fullbench],
+    timeout: 600) # Timeout should work on HDD drive
+endif
+
+if host_machine_os != os_windows
+  playTests_sh = find_program(join_paths(zstd_rootdir, 'tests/playTests.sh'), required: true)
+  test('test-zstd',
+    playTests_sh,
+    args: ZSTDRTTEST,
+    env: ['ZSTD=' + zstd.full_path()],
+    depends: [datagen],
+    timeout: 600) # Timeout should work on HDD drive
+endif
+
+test('test-fullbench-1',
+  fullbench,
+  args: ['-i1'],
+  depends: [datagen],
+  timeout: 60)
+test('test-fullbench-2',
+  fullbench,
+  args: ['-i1', '-P0'],
+  depends: [datagen],
+  timeout: 60)
+
+if use_zlib
+  test('test-fuzzer',
+    fuzzer,
+    args: ['-v', FUZZERTEST] + FUZZER_FLAGS,
+    timeout: 240)
+endif
+
+test('test-zbuff',
+  zbufftest,
+  args: [ZSTREAM_TESTTIME],
+  timeout: 120)
+test('test-zstream-1',
+  zstreamtest,
+  args: ['-v', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
+  timeout: 120)
+test('test-zstream-2',
+  zstreamtest,
+  args: ['-mt', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
+  timeout: 120)
+test('test-zstream-3',
+  zstreamtest,
+  args: ['--newapi', '-t1', ZSTREAM_TESTTIME] + FUZZER_FLAGS,
+  timeout: 120)
+test('test-longmatch', longmatch, timeout: 36)
+test('test-invalidDictionaries', invalidDictionaries) # should be fast
+test('test-symbols', symbols) # should be fast
+test('test-legacy', legacy) # should be fast
+test('test-decodecorpus',
+  decodecorpus,
+  args: ['-t', DECODECORPUS_TESTTIME],
+  timeout: 60)
+test('test-poolTests', poolTests) # should be fast

build/meson/tests/valgrindTest.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# #############################################################################
+# Copyright (c) 2018-present    lzutao <taolzu(at)gmail.com>
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# #############################################################################
+import os
+import subprocess
+import tempfile
+
+
+def valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench):
+  VALGRIND_ARGS = [valgrind, '--leak-check=full', '--show-leak-kinds=all', '--error-exitcode=1']
+
+  print('\n ---- valgrind tests : memory analyzer ----')
+
+  subprocess.check_call([*VALGRIND_ARGS, datagen, '-g50M'], stdout=subprocess.DEVNULL)
+
+  if subprocess.call([*VALGRIND_ARGS, zstd],
+                     stdout=subprocess.DEVNULL) == 0:
+    raise subprocess.CalledProcessError('zstd without argument should have failed')
+
+  with subprocess.Popen([datagen, '-g80'], stdout=subprocess.PIPE) as p1, \
+       subprocess.Popen([*VALGRIND_ARGS, zstd, '-', '-c'],
+                        stdin=p1.stdout,
+                        stdout=subprocess.DEVNULL) as p2:
+    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
+    p2.communicate()
+    if p2.returncode != 0:
+      raise subprocess.CalledProcessError()
+
+  with subprocess.Popen([datagen, '-g16KB'], stdout=subprocess.PIPE) as p1, \
+       subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
+                        stdin=p1.stdout,
+                        stdout=subprocess.DEVNULL) as p2:
+    p1.stdout.close()
+    p2.communicate()
+    if p2.returncode != 0:
+      raise subprocess.CalledProcessError()
+
+  with tempfile.NamedTemporaryFile() as tmp_fd:
+    with subprocess.Popen([datagen, '-g2930KB'], stdout=subprocess.PIPE) as p1, \
+         subprocess.Popen([*VALGRIND_ARGS, zstd, '-5', '-vf', '-', '-o', tmp_fd.name],
+                          stdin=p1.stdout) as p2:
+      p1.stdout.close()
+      p2.communicate()
+      if p2.returncode != 0:
+        raise subprocess.CalledProcessError()
+
+    subprocess.check_call([*VALGRIND_ARGS, zstd, '-vdf', tmp_fd.name, '-c'],
+                          stdout=subprocess.DEVNULL)
+
+    with subprocess.Popen([datagen, '-g64MB'], stdout=subprocess.PIPE) as p1, \
+         subprocess.Popen([*VALGRIND_ARGS, zstd, '-vf', '-', '-c'],
+                          stdin=p1.stdout,
+                          stdout=subprocess.DEVNULL) as p2:
+      p1.stdout.close()
+      p2.communicate()
+      if p2.returncode != 0:
+        raise subprocess.CalledProcessError()
+
+  subprocess.check_call([*VALGRIND_ARGS, fuzzer, '-T1mn', '-t1'])
+  subprocess.check_call([*VALGRIND_ARGS, fullbench, '-i1'])
+
+
+def main():
+  import argparse
+  parser = argparse.ArgumentParser(description='Valgrind tests : memory analyzer')
+  parser.add_argument('valgrind', help='valgrind path')
+  parser.add_argument('zstd', help='zstd path')
+  parser.add_argument('datagen', help='datagen path')
+  parser.add_argument('fuzzer', help='fuzzer path')
+  parser.add_argument('fullbench', help='fullbench path')
+
+  args = parser.parse_args()
+
+  valgrind = args.valgrind
+  zstd = args.zstd
+  datagen = args.datagen
+  fuzzer = args.fuzzer
+  fullbench = args.fullbench
+
+  valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench)
+
+
+if __name__ == '__main__':
+  main()

contrib/meson/README

@@ -1,3 +0,0 @@
-This Meson project is provided with no guarantee and maintained by Dima Krasner <dima@dimakrasner.com>.
-
-It outputs one libzstd, either shared or static, depending on default_library.

contrib/meson/meson.build

@@ -1,144 +0,0 @@
-project('zstd', 'c', license: 'BSD')
-
-libm = meson.get_compiler('c').find_library('m', required: true)
-
-lib_dir = join_paths('..', '..', 'lib')
-common_dir = join_paths(lib_dir, 'common')
-compress_dir = join_paths(lib_dir, 'compress')
-decompress_dir = join_paths(lib_dir, 'decompress')
-dictbuilder_dir = join_paths(lib_dir, 'dictBuilder')
-deprecated_dir = join_paths(lib_dir, 'deprecated')
-
-libzstd_srcs = [
-    join_paths(common_dir, 'entropy_common.c'),
-    join_paths(common_dir, 'fse_decompress.c'),
-    join_paths(common_dir, 'threading.c'),
-    join_paths(common_dir, 'pool.c'),
-    join_paths(common_dir, 'zstd_common.c'),
-    join_paths(common_dir, 'error_private.c'),
-    join_paths(common_dir, 'xxhash.c'),
-    join_paths(compress_dir, 'fse_compress.c'),
-    join_paths(compress_dir, 'hist.c'),
-    join_paths(compress_dir, 'huf_compress.c'),
-    join_paths(compress_dir, 'zstd_compress.c'),
-    join_paths(compress_dir, 'zstd_fast.c'),
-    join_paths(compress_dir, 'zstd_double_fast.c'),
-    join_paths(compress_dir, 'zstd_lazy.c'),
-    join_paths(compress_dir, 'zstd_opt.c'),
-    join_paths(compress_dir, 'zstd_ldm.c'),
-    join_paths(compress_dir, 'zstdmt_compress.c'),
-    join_paths(decompress_dir, 'huf_decompress.c'),
-    join_paths(decompress_dir, 'zstd_decompress.c'),
-    join_paths(dictbuilder_dir, 'cover.c'),
-    join_paths(dictbuilder_dir, 'divsufsort.c'),
-    join_paths(dictbuilder_dir, 'zdict.c'),
-    join_paths(deprecated_dir, 'zbuff_common.c'),
-    join_paths(deprecated_dir, 'zbuff_compress.c'),
-    join_paths(deprecated_dir, 'zbuff_decompress.c')
-]
-
-libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)]
-
-legacy = get_option('legacy_support')
-if legacy == '0'
-    legacy = 'false'
-endif
-if legacy != 'false'
-    if legacy == 'true'
-        legacy = '1'
-    endif
-    #See ZSTD_LEGACY_SUPPORT of programs/README.md
-    message('Enabling legacy support back to version 0.' + legacy)
-    legacy_int = legacy.to_int()
-    if legacy_int > 7
-        legacy_int = 7
-    endif
-    libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=' + legacy]
-
-    legacy_dir = join_paths(lib_dir, 'legacy')
-    libzstd_includes += [include_directories(legacy_dir)]
-    if legacy_int <= 1
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v01.c')
-    endif
-    if legacy_int <= 2
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v02.c')
-    endif
-    if legacy_int <= 3
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v03.c')
-    endif
-    if legacy_int <= 4
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v04.c')
-    endif
-    if legacy_int <= 5
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v05.c')
-    endif
-    if legacy_int <= 6
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v06.c')
-    endif
-    if legacy_int <= 7
-        libzstd_srcs += join_paths(legacy_dir, 'zstd_v07.c')
-    endif
-else
-    libzstd_cflags = []
-endif
-
-if get_option('multithread')
-    message('Enabling multi-threading support')
-    add_global_arguments('-DZSTD_MULTITHREAD', language: 'c')
-    libzstd_deps = [dependency('threads')]
-else
-    libzstd_deps = []
-endif
-
-libzstd = library('zstd',
-                  libzstd_srcs,
-                  include_directories: libzstd_includes,
-                  c_args: libzstd_cflags,
-                  dependencies: libzstd_deps,
-                  install: true,
-                  soversion: '1',
-                 )
-
-programs_dir = join_paths('..', '..', 'programs')
-
-zstd = executable('zstd',
-                  join_paths(programs_dir, 'bench.c'),
-                  join_paths(programs_dir, 'datagen.c'),
-                  join_paths(programs_dir, 'dibio.c'),
-                  join_paths(programs_dir, 'fileio.c'),
-                  join_paths(programs_dir, 'zstdcli.c'),
-                  include_directories: libzstd_includes,
-                  c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'],
-                  link_with: libzstd,
-                  install: true)
-
-tests_dir = join_paths('..', '..', 'tests')
-datagen_c = join_paths(programs_dir, 'datagen.c')
-test_includes = libzstd_includes + [include_directories(programs_dir)]
-
-fullbench = executable('fullbench',
-                       datagen_c, join_paths(tests_dir, 'fullbench.c'),
-                       include_directories: test_includes,
-                       link_with: libzstd)
-test('fullbench', fullbench)
-
-fuzzer = executable('fuzzer',
-                    datagen_c, join_paths(tests_dir, 'fuzzer.c'),
-                    include_directories: test_includes,
-                    link_with: libzstd)
-test('fuzzer', fuzzer)
-
-if target_machine.system() != 'windows'
-    paramgrill = executable('paramgrill',
-                            datagen_c, join_paths(tests_dir, 'paramgrill.c'),
-                            join_paths(programs_dir, 'bench.c'),
-                            include_directories: test_includes,
-                            link_with: libzstd,
-                            dependencies: libm)
-    test('paramgrill', paramgrill)
-
-    datagen = executable('datagen',
-                         datagen_c, join_paths(tests_dir, 'datagencli.c'),
-                         include_directories: test_includes,
-                         link_with: libzstd)
-endif

contrib/meson/meson_options.txt

@@ -1,3 +0,0 @@
-option('multithread', type: 'boolean', value: false)
-option('legacy_support', type: 'string', value: '4',
-    description: 'True or false, or 7 to 1 for v0.7+ to v0.1+.')

doc/zstd_manual.html

@@ -487,8 +487,9 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
     </b>/* frame parameters */<b>
     ZSTD_c_contentSizeFlag=200, </b>/* Content size will be written into frame header _whenever known_ (default:1)<b>
-                              * Content size must be known at the beginning of compression,
+                              * Content size must be known at the beginning of compression.
-                              * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
+                              * This is automatically the case when using ZSTD_compress2(),
+                              * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
     ZSTD_c_checksumFlag=201, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
     ZSTD_c_dictIDFlag=202,   </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>
 
@@ -524,12 +525,13 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
      * ZSTD_c_forceMaxWindow
      * ZSTD_c_forceAttachDict
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
-     * note : never ever use experimentalParam? names directly
+     * note : never ever use experimentalParam? names directly;
+     *        also, the enums values themselves are unstable and can still change.
      */
      ZSTD_c_experimentalParam1=500,
      ZSTD_c_experimentalParam2=10,
      ZSTD_c_experimentalParam3=1000,
-     ZSTD_c_experimentalParam4
+     ZSTD_c_experimentalParam4=1001
 } ZSTD_cParameter;
 </b></pre><BR>
 <pre><b>typedef struct {
@@ -562,14 +564,17 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
 <pre><b>size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
 </b><p>  Total input data size to be compressed as a single frame.
-  This value will be controlled at end of frame, and trigger an error if not respected.
+  Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
+  This value will also be controlled at end of frame, and trigger an error if not respected.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
            In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
            ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
   Note 2 : pledgedSrcSize is only valid once, for the next frame.
-           It's discarded at the end of the frame.
+           It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
-  Note 3 : If all data is provided and consumed in a single round,
+  Note 3 : Whenever all input data is provided and consumed in a single round,
+           for example with ZSTD_compress2(),
+           or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
            this value is automatically overriden by srcSize instead.
  
 </p></pre><BR>
@@ -652,6 +657,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
                        void* dst, size_t dstCapacity,
                  const void* src, size_t srcSize);
 </b><p>  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+  ZSTD_compress2() always starts a new frame.
+  Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
   - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
   - The function is always blocking, returns when compression is completed.
   Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.

examples/dictionary_compression.c

@@ -7,71 +7,13 @@
  * in the COPYING file in the root directory of this source tree).
  * You may select, at your option, one of the above-listed licenses.
  */
-
-
 #include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // printf
 #include <string.h>    // strerror
 #include <errno.h>     // errno
 #include <sys/stat.h>  // stat
 #include <zstd.h>      // presumes zstd library is installed
-
+#include "utils.h"
-
-static off_t fsize_orDie(const char *filename)
-{
-    struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
-    /* error */
-    perror(filename);
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(2);
-}
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc");
-    exit(3);
-}
-
-static void* loadFile_orDie(const char* fileName, size_t* size)
-{
-    off_t const buffSize = fsize_orDie(fileName);
-    FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
-        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
-        exit(4);
-    }
-    fclose(inFile);
-    *size = buffSize;
-    return buffer;
-}
-
-static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
-{
-    FILE* const oFile = fopen_orDie(fileName, "wb");
-    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
-    if (wSize != (size_t)buffSize) {
-        fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
-        exit(5);
-    }
-    if (fclose(oFile)) {
-        perror(fileName);
-        exit(6);
-    }
-}
 
 /* createDict() :
    `dictFileName` is supposed to have been created using `zstd --train` */

examples/dictionary_decompression.c

@@ -17,49 +17,7 @@
 #include <sys/stat.h>  // stat
 #define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
 #include <zstd.h>      // presumes zstd library is installed
-
+#include "utils.h"
-
-static off_t fsize_orDie(const char *filename)
-{
-    struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
-    /* error */
-    perror(filename);
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(2);
-}
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc");
-    exit(3);
-}
-
-static void* loadFile_orDie(const char* fileName, size_t* size)
-{
-    off_t const buffSize = fsize_orDie(fileName);
-    FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
-        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
-        exit(4);
-    }
-    fclose(inFile);
-    *size = buffSize;
-    return buffer;
-}
 
 /* createDict() :
    `dictFileName` is supposed to have been created using `zstd --train` */
@@ -74,7 +32,6 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
     return ddict;
 }
 
-
 static void decompress(const char* fname, const ZSTD_DDict* ddict)
 {
     size_t cSize;

examples/multiple_streaming_compression.c

@@ -20,53 +20,7 @@
 #include <errno.h>     // errno
 #define ZSTD_STATIC_LINKING_ONLY  // streaming API defined as "experimental" for the time being
 #include <zstd.h>      // presumes zstd library is installed
-
+#include "utils.h"
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc:");
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(3);
-}
-
-static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
-{
-    size_t const readSize = fread(buffer, 1, sizeToRead, file);
-    if (readSize == sizeToRead) return readSize;   /* good */
-    if (feof(file)) return readSize;   /* good, reached end of file */
-    /* error */
-    perror("fread");
-    exit(4);
-}
-
-static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
-{
-    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
-    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
-    /* error */
-    perror("fwrite");
-    exit(5);
-}
-
-static size_t fclose_orDie(FILE* file)
-{
-    if (!fclose(file)) return 0;
-    /* error */
-    perror("fclose");
-    exit(6);
-}
-
 
 typedef struct {
     void* buffIn;
@@ -95,7 +49,6 @@ static void freeResources(resources ress)
     free(ress.buffOut);
 }
 
-
 static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel)
 {
     FILE* const fin  = fopen_orDie(fname, "rb");
@@ -125,7 +78,6 @@ static void compressFile_orDie(resources ress, const char* fname, const char* ou
     fclose_orDie(fin);
 }
 
-
 int main(int argc, const char** argv)
 {
     const char* const exeName = argv[0];

examples/simple_compression.c

@@ -8,78 +8,13 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-
-
 #include <stdlib.h>    // malloc, free, exit
 #include <stdio.h>     // fprintf, perror, fopen, etc.
 #include <string.h>    // strlen, strcat, memset, strerror
 #include <errno.h>     // errno
 #include <sys/stat.h>  // stat
 #include <zstd.h>      // presumes zstd library is installed
-
+#include "utils.h"
-
-static off_t fsize_orDie(const char *filename)
-{
-    struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
-    /* error */
-    perror(filename);
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(2);
-}
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror(NULL);
-    exit(3);
-}
-
-static void* loadFile_orDie(const char* fileName, size_t* size)
-{
-    off_t const fileSize = fsize_orDie(fileName);
-    size_t const buffSize = (size_t)fileSize;
-    if ((off_t)buffSize < fileSize) {   /* narrowcast overflow */
-        fprintf(stderr, "%s : filesize too large \n", fileName);
-        exit(4);
-    }
-    FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
-        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
-        exit(5);
-    }
-    fclose(inFile);  /* can't fail, read only */
-    *size = buffSize;
-    return buffer;
-}
-
-
-static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
-{
-    FILE* const oFile = fopen_orDie(fileName, "wb");
-    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
-    if (wSize != (size_t)buffSize) {
-        fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
-        exit(6);
-    }
-    if (fclose(oFile)) {
-        perror(fileName);
-        exit(7);
-    }
-}
-
 
 static void compress_orDie(const char* fname, const char* oname)
 {
@@ -103,7 +38,6 @@ static void compress_orDie(const char* fname, const char* oname)
     free(cBuff);
 }
 
-
 static char* createOutFilename_orDie(const char* filename)
 {
     size_t const inL = strlen(filename);

examples/simple_decompression.c

@@ -15,50 +15,7 @@
 #include <sys/stat.h>  // stat
 #define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
 #include <zstd.h>      // presumes zstd library is installed
-
+#include "utils.h"
-
-static off_t fsize_orDie(const char *filename)
-{
-    struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
-    /* error */
-    fprintf(stderr, "stat: %s : %s \n", filename, strerror(errno));
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    fprintf(stderr, "fopen: %s : %s \n", filename, strerror(errno));
-    exit(2);
-}
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size + !size);   /* avoid allocating size of 0 : may return NULL (implementation dependent) */
-    if (buff) return buff;
-    /* error */
-    fprintf(stderr, "malloc: %s \n", strerror(errno));
-    exit(3);
-}
-
-static void* loadFile_orDie(const char* fileName, size_t* size)
-{
-    off_t const buffSize = fsize_orDie(fileName);
-    FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
-        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
-        exit(4);
-    }
-    fclose(inFile);   /* can't fail (read only) */
-    *size = buffSize;
-    return buffer;
-}
-
 
 static void decompress(const char* fname)
 {
@@ -90,7 +47,6 @@ static void decompress(const char* fname)
     free(cBuff);
 }
 
-
 int main(int argc, const char** argv)
 {
     const char* const exeName = argv[0];

examples/streaming_compression.c

@@ -13,53 +13,7 @@
 #include <stdio.h>     // fprintf, perror, feof, fopen, etc.
 #include <string.h>    // strlen, memset, strcat
 #include <zstd.h>      // presumes zstd library is installed
-
+#include "utils.h"
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc:");
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(3);
-}
-
-static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
-{
-    size_t const readSize = fread(buffer, 1, sizeToRead, file);
-    if (readSize == sizeToRead) return readSize;   /* good */
-    if (feof(file)) return readSize;   /* good, reached end of file */
-    /* error */
-    perror("fread");
-    exit(4);
-}
-
-static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
-{
-    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
-    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
-    /* error */
-    perror("fwrite");
-    exit(5);
-}
-
-static size_t fclose_orDie(FILE* file)
-{
-    if (!fclose(file)) return 0;
-    /* error */
-    perror("fclose");
-    exit(6);
-}
-
 
 static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
 {
@@ -102,8 +56,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
 
     ZSTD_freeCStream(cstream);
     fclose_orDie(fout);
-    fclose_orDie(fin);
+    fclose_orDie(fin);    free(buffIn);
-    free(buffIn);
     free(buffOut);
 }
 

examples/streaming_decompression.c

@@ -14,53 +14,7 @@
 #include <string.h>    // strerror
 #include <errno.h>     // errno
 #include <zstd.h>      // presumes zstd library is installed
-
+#include "utils.h"
-
-static void* malloc_orDie(size_t size)
-{
-    void* const buff = malloc(size);
-    if (buff) return buff;
-    /* error */
-    perror("malloc:");
-    exit(1);
-}
-
-static FILE* fopen_orDie(const char *filename, const char *instruction)
-{
-    FILE* const inFile = fopen(filename, instruction);
-    if (inFile) return inFile;
-    /* error */
-    perror(filename);
-    exit(3);
-}
-
-static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
-{
-    size_t const readSize = fread(buffer, 1, sizeToRead, file);
-    if (readSize == sizeToRead) return readSize;   /* good */
-    if (feof(file)) return readSize;   /* good, reached end of file */
-    /* error */
-    perror("fread");
-    exit(4);
-}
-
-static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
-{
-    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
-    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
-    /* error */
-    perror("fwrite");
-    exit(5);
-}
-
-static size_t fclose_orDie(FILE* file)
-{
-    if (!fclose(file)) return 0;
-    /* error */
-    perror("fclose");
-    exit(6);
-}
-
 
 static void decompressFile_orDie(const char* fname)
 {

examples/utils.h

@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*
+ * This header file has common utility functions used in examples. 
+ */
+#ifndef UTILS_H
+#define UTILS_H
+
+#include <stdlib.h>    // malloc, free, exit
+#include <stdio.h>     // fprintf, perror, fopen, etc.
+#include <string.h>    // strlen, strcat, memset, strerror
+#include <errno.h>     // errno
+#include <sys/stat.h>  // stat
+
+/*
+ * Define the returned error code from utility functions. 
+ */
+typedef enum {
+    ERROR_fsize = 1,
+    ERROR_fopen = 2,
+    ERROR_fclose = 3,
+    ERROR_fread = 4,
+    ERROR_fwrite = 5,
+    ERROR_loadFile = 6,
+    ERROR_saveFile = 7,
+    ERROR_malloc = 8,
+    ERROR_largeFile = 9,
+} UTILS_ErrorCode;
+
+/*! fsize_orDie() : 
+ * Get the size of a given file path.
+ * 
+ * @return The size of a given file path.
+ */
+static off_t fsize_orDie(const char *filename)
+{
+    struct stat st;
+    if (stat(filename, &st) == 0) return st.st_size;
+    /* error */
+    perror(filename);
+    exit(ERROR_fsize);
+}
+
+/*! fopen_orDie() : 
+ * Open a file using given file path and open option.
+ *
+ * @return If successful this function will return a FILE pointer to an
+ * opened file otherwise it sends an error to stderr and exits.
+ */
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(ERROR_fopen);
+}
+
+/*! fclose_orDie() : 
+ * Close an opened file using given FILE pointer.
+ */
+static void fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) { return; };
+    /* error */
+    perror("fclose");
+    exit(ERROR_fclose);
+}
+
+/*! fread_orDie() : 
+ * 
+ * Read sizeToRead bytes from a given file, storing them at the
+ * location given by buffer.
+ * 
+ * @return The number of bytes read.
+ */
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(ERROR_fread);
+}
+
+/*! fwrite_orDie() :
+ *  
+ * Write sizeToWrite bytes to a file pointed to by file, obtaining
+ * them from a location given by buffer.
+ *
+ * Note: This function will send an error to stderr and exit if it
+ * cannot write data to the given file pointer. 
+ *
+ * @return The number of bytes written.
+ */
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(ERROR_fwrite);
+}
+
+/*! malloc_orDie() :
+ * Allocate memory.
+ * 
+ * @return If successful this function returns a pointer to allo-
+ * cated memory.  If there is an error, this function will send that
+ * error to stderr and exit.
+ */
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc");
+    exit(ERROR_malloc);
+}
+
+/*! loadFile_orDie() :
+ * Read size bytes from a file.
+ * 
+ * Note: This function will send an error to stderr and exit if it
+ * cannot read data from the given file path.
+ * 
+ * @return If successful this function will return a pointer to read
+ * data otherwise it will printout an error to stderr and exit.
+ */
+static void* loadFile_orDie(const char* fileName, size_t* size)
+{
+    off_t const fileSize = fsize_orDie(fileName);
+    size_t const buffSize = (size_t)fileSize;
+    if ((off_t)buffSize < fileSize) {   /* narrowcast overflow */
+        fprintf(stderr, "%s : filesize too large \n", fileName);
+        exit(ERROR_largeFile);
+    }
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    void* const buffer = malloc_orDie(buffSize);
+    size_t const readSize = fread(buffer, 1, buffSize, inFile);
+    if (readSize != (size_t)buffSize) {
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
+        exit(ERROR_fread);
+    }
+    fclose(inFile);  /* can't fail, read only */
+    *size = buffSize;
+    return buffer;
+}
+
+/*! saveFile_orDie() :
+ *  
+ * Save buffSize bytes to a given file path, obtaining them from a location pointed
+ * to by buff.
+ * 
+ * Note: This function will send an error to stderr and exit if it
+ * cannot write to a given file.
+ */
+static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
+{
+    FILE* const oFile = fopen_orDie(fileName, "wb");
+    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
+    if (wSize != (size_t)buffSize) {
+        fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
+        exit(ERROR_fwrite);
+    }
+    if (fclose(oFile)) {
+        perror(fileName);
+        exit(ERROR_fclose);
+    }
+}
+
+#endif

lib/BUCK

@@ -1,6 +1,7 @@
 cxx_library(
     name='zstd',
     header_namespace='',
+    exported_headers=['zstd.h'],
     visibility=['PUBLIC'],
     deps=[
         ':common',
@@ -17,7 +18,7 @@ cxx_library(
     exported_headers=subdir_glob([
         ('compress', 'zstd*.h'),
     ]),
-    srcs=glob(['compress/zstd*.c']),
+    srcs=glob(['compress/zstd*.c', 'compress/hist.c']),
     deps=[':common'],
 )
 
@@ -40,7 +41,7 @@ cxx_library(
     header_namespace='',
     visibility=['PUBLIC'],
     exported_headers=subdir_glob([
-        ('decprecated', '*.h'),
+        ('deprecated', '*.h'),
     ]),
     srcs=glob(['deprecated/*.c']),
     deps=[':common'],
@@ -118,6 +119,7 @@ cxx_library(
         'decompress/huf_decompress.c',
     ],
     deps=[
+        ':debug',
         ':bitstream',
         ':compiler',
         ':errors',
@@ -204,9 +206,20 @@ cxx_library(
     ],
 )
 
+cxx_library(
+    name='debug',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'debug.h'),
+    ]),
+    srcs=['common/debug.c'],
+)
+
 cxx_library(
     name='common',
     deps=[
+        ':debug',
         ':bitstream',
         ':compiler',
         ':cpu',

lib/Makefile

@@ -31,7 +31,12 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
-GREP = grep --color=never
+HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+GREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+GREP_OPTIONS += --color=never
+endif
+GREP = grep $(GREP_OPTIONS)
 
 ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
 ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))

lib/common/cpu.h

@@ -78,7 +78,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
       __asm__(
           "pushl %%ebx\n\t"
           "cpuid\n\t"
-          "movl %%ebx, %%eax\n\r"
+          "movl %%ebx, %%eax\n\t"
           "popl %%ebx"
           : "=a"(f7b), "=c"(f7c)
           : "a"(7), "c"(0)

lib/compress/zstd_compress.c

@@ -4091,19 +4091,21 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
                       void* dst, size_t dstCapacity,
                       const void* src, size_t srcSize)
 {
-    size_t oPos = 0;
+    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
-    size_t iPos = 0;
+    {   size_t oPos = 0;
-    size_t const result = ZSTD_compressStream2_simpleArgs(cctx,
+        size_t iPos = 0;
-                                    dst, dstCapacity, &oPos,
+        size_t const result = ZSTD_compressStream2_simpleArgs(cctx,
-                                    src, srcSize, &iPos,
+                                        dst, dstCapacity, &oPos,
-                                    ZSTD_e_end);
+                                        src, srcSize, &iPos,
-    assert(iPos == srcSize);
+                                        ZSTD_e_end);
-    if (ZSTD_isError(result)) return result;
+        if (ZSTD_isError(result)) return result;
-    if (result != 0) {  /* compression not completed, due to lack of output space */
+        if (result != 0) {  /* compression not completed, due to lack of output space */
-        assert(oPos == dstCapacity);
+            assert(oPos == dstCapacity);
-        return ERROR(dstSize_tooSmall);
+            return ERROR(dstSize_tooSmall);
+        }
+        assert(iPos == srcSize);   /* all input is expected consumed */
+        return oPos;
     }
-    return oPos;
 }
 
 /*======   Finalize   ======*/

lib/zstd.h

@@ -583,8 +583,9 @@ typedef enum {
 
     /* frame parameters */
     ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
-                              * Content size must be known at the beginning of compression,
+                              * Content size must be known at the beginning of compression.
-                              * it is provided using ZSTD_CCtx_setPledgedSrcSize() */
+                              * This is automatically the case when using ZSTD_compress2(),
+                              * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
     ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_c_dictIDFlag=202,   /* When applicable, dictionary's ID is written into frame header (default:1) */
 
@@ -620,12 +621,13 @@ typedef enum {
      * ZSTD_c_forceMaxWindow
      * ZSTD_c_forceAttachDict
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
-     * note : never ever use experimentalParam? names directly
+     * note : never ever use experimentalParam? names directly;
+     *        also, the enums values themselves are unstable and can still change.
      */
      ZSTD_c_experimentalParam1=500,
      ZSTD_c_experimentalParam2=10,
      ZSTD_c_experimentalParam3=1000,
-     ZSTD_c_experimentalParam4
+     ZSTD_c_experimentalParam4=1001
 } ZSTD_cParameter;
 
 
@@ -659,14 +661,17 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
 
 /*! ZSTD_CCtx_setPledgedSrcSize() :
  *  Total input data size to be compressed as a single frame.
- *  This value will be controlled at end of frame, and trigger an error if not respected.
+ *  Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
+ *  This value will also be controlled at end of frame, and trigger an error if not respected.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
  *           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
  *           ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
  *  Note 2 : pledgedSrcSize is only valid once, for the next frame.
- *           It's discarded at the end of the frame.
+ *           It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
- *  Note 3 : If all data is provided and consumed in a single round,
+ *  Note 3 : Whenever all input data is provided and consumed in a single round,
+ *           for example with ZSTD_compress2(),
+ *           or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
  *           this value is automatically overriden by srcSize instead.
  */
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
@@ -750,6 +755,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
 
 /*! ZSTD_compress2() :
  *  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
+ *  ZSTD_compress2() always starts a new frame.
+ *  Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
  *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
  *  - The function is always blocking, returns when compression is completed.
  *  Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.

programs/Makefile

@@ -29,7 +29,12 @@ LIBVER  := $(shell echo $(LIBVER_SCRIPT))
 
 ZSTD_VERSION = $(LIBVER)
 
-GREP = grep --color=never
+HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+GREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+GREP_OPTIONS += --color=never
+endif
+GREP = grep $(GREP_OPTIONS)
 
 ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
 ALIGN_LOOP = -falign-loops=32
@@ -91,7 +96,7 @@ VOID = /dev/null
 
 # thread detection
 NO_THREAD_MSG := ==> no threads, building without multithreading support
-HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_pthread$(EXT) -x c - -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0)
+HAVE_PTHREAD := $(shell printf '\#include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
 HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
 ifeq ($(HAVE_THREAD), 1)
 THREAD_MSG := ==> building with threading support
@@ -103,7 +108,7 @@ endif
 
 # zlib detection
 NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
-HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_zlib$(EXT) -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0)
+HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
 ifeq ($(HAVE_ZLIB), 1)
 ZLIB_MSG := ==> building zstd with .gz compression support
 ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
@@ -114,7 +119,7 @@ endif
 
 # lzma detection
 NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
-HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lzma$(EXT) -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0)
+HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
 ifeq ($(HAVE_LZMA), 1)
 LZMA_MSG := ==> building zstd with .xz/.lzma compression support
 LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
@@ -125,7 +130,7 @@ endif
 
 # lz4 detection
 NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
-HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' | $(CC) $(FLAGS) -o have_lz4$(EXT) -x c - -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0)
+HAVE_LZ4 := $(shell printf '\#include <lz4frame.h>\n\#include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
 ifeq ($(HAVE_LZ4), 1)
 LZ4_MSG := ==> building zstd with .lz4 compression support
 LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
@@ -275,7 +280,12 @@ preview-man: clean-man man
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
 
-EGREP = egrep --color=never
+HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+EGREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+EGREP_OPTIONS += --color=never
+endif
+EGREP = egrep $(EGREP_OPTIONS)
 
 # Print a two column output of targets and their description. To add a target description, put a
 # comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>".  For example:

programs/zstdcli.c

@@ -151,7 +151,7 @@ static int usage_advanced(const char* programName)
 #ifdef UTIL_HAS_CREATEFILELIST
     DISPLAY( " -r     : operate recursively on directories \n");
 #endif
-    DISPLAY( "--format=zstd : compress files to the .zstd format (default) \n");
+    DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
 #ifdef ZSTD_GZCOMPRESS
     DISPLAY( "--format=gzip : compress files to the .gz format \n");
 #endif

programs/zstdgrep

@@ -31,94 +31,101 @@ grep_args=""
 hyphen=0
 silent=0
 
-prg=$(basename $0)
+prg=$(basename "$0")
 
 # handle being called 'zegrep' or 'zfgrep'
-case ${prg} in
+case "${prg}" in
-    *zegrep)
+    *zegrep) grep_args="-E";;
-	grep_args="-E";;
+    *zfgrep) grep_args="-F";;
-    *zfgrep)
-	grep_args="-F";;
 esac
 
 # skip all options and pass them on to grep taking care of options
 # with arguments, and if -e was supplied
 
-while [ $# -gt 0 -a ${endofopts} -eq 0 ]
+while [ "$#" -gt 0 ] && [ "${endofopts}" -eq 0 ]; do
-do
+    case "$1" in
-    case $1 in
     # from GNU grep-2.5.1 -- keep in sync!
-	-[ABCDXdefm])
+        -[ABCDXdefm])
-	    if [ $# -lt 2 ]
+            if [ "$#" -lt 2 ]; then
-		then
+                printf '%s: missing argument for %s flag\n' "${prg}" "$1" >&2
-		echo "${prg}: missing argument for $1 flag" >&2
+                exit 1
-		exit 1
+            fi
-	    fi
+            case "$1" in
-	    case $1 in
+                -e)
-		-e)
+                    pattern="$2"
-		    pattern="$2"
+                    pattern_found=1
-		    pattern_found=1
+                    shift 2
-		    shift 2
+                    break
-		    break
+                    ;;
-		    ;;
+                *)
-		*)
+                    ;;
-		    ;;
+            esac
-	    esac
+            grep_args="${grep_args} $1 $2"
-	    grep_args="${grep_args} $1 $2"
+            shift 2
-	    shift 2
+            ;;
-	    ;;
+        --)
-	--)
+            shift
-	    shift
+            endofopts=1
-	    endofopts=1
+            ;;
-	    ;;
+        -)
-	-)
+            hyphen=1
-	    hyphen=1
+            shift
-	    shift
+            ;;
-	    ;;
+        -h)
-	-h)
+            silent=1
-	    silent=1
+            shift
-	    shift
+            ;;
-	    ;;
+        -*)
-	-*)
+            grep_args="${grep_args} $1"
-	    grep_args="${grep_args} $1"
+            shift
-	    shift
+            ;;
-	    ;;
+        *)
-	*)
+            # pattern to grep for
-	    # pattern to grep for
+            endofopts=1
-	    endofopts=1
+            ;;
-	    ;;
     esac
 done
 
 # if no -e option was found, take next argument as grep-pattern
-if [ ${pattern_found} -lt 1 ]
+if [ "${pattern_found}" -lt 1 ]; then
-then
+    if [ "$#" -ge 1 ]; then
-    if [ $# -ge 1 ]; then
+        pattern="$1"
-	pattern="$1"
+        shift
-	shift
+    elif [ "${hyphen}" -gt 0 ]; then
-    elif [ ${hyphen} -gt 0 ]; then
+        pattern="-"
-	pattern="-"
     else
-	echo "${prg}: missing pattern" >&2
+        printf '%s: missing pattern\n' "${prg}" >&2
-	exit 1
+        exit 1
     fi
 fi
 
+EXIT_CODE=0
 # call grep ...
-if [ $# -lt 1 ]
+if [ "$#" -lt 1 ]; then
-then
     # ... on stdin
-    ${zcat} -fq - | ${grep} ${grep_args} -- "${pattern}" -
+    set -f # Disable file name generation (globbing).
+    # shellcheck disable=SC2086
+    "${zcat}" -fq - | "${grep}" ${grep_args} -- "${pattern}" -
+    EXIT_CODE=$?
+    set +f
 else
     # ... on all files given on the command line
-    if [ ${silent} -lt 1 -a $# -gt 1 ]; then
+    if [ "${silent}" -lt 1 ] && [ "$#" -gt 1 ]; then
-	grep_args="-H ${grep_args}"
+        grep_args="-H ${grep_args}"
     fi
-    while [ $# -gt 0 ]
+    CUR_EXIT_CODE=0
-    do
+    EXIT_CODE=1
-	${zcat} -fq -- "$1" | ${grep} --label="${1}" ${grep_args} -- "${pattern}" -
+    set -f
-	shift
+    while [ "$#" -gt 0 ]; do
+        # shellcheck disable=SC2086
+        "${zcat}" -fq -- "$1" | "${grep}" --label="${1}" ${grep_args} -- "${pattern}" -
+        CUR_EXIT_CODE=$?
+        if [ "${CUR_EXIT_CODE}" -eq 0 ] && [ "${EXIT_CODE}" -ne 1 ]; then
+            EXIT_CODE=0
+        fi
+        shift
     done
+    set +f
 fi
 
-exit 0
+exit "${EXIT_CODE}"

tests/Makefile

@@ -320,7 +320,7 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
 test-all: test test32 valgrindTest test-decodecorpus-cli
 
 
-.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
+.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep
 test-zstd: ZSTD = $(PRGDIR)/zstd
 test-zstd: zstd
 
@@ -352,6 +352,10 @@ test-gzstd: gzstd
 	$(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz
 	$(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt
 
+test-zstdgrep: gzstd
+	@echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep a
+	@echo a | $(PRGDIR)/zstd | $(PRGDIR)/zstdgrep b && return 1 || return 0
+
 test-fullbench: fullbench datagen
 	$(QEMU_SYS) ./fullbench -i1
 	$(QEMU_SYS) ./fullbench -i1 -P0

tests/regression/Makefile

@@ -0,0 +1,58 @@
+# ################################################################
+# Copyright (c) 2015-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+CFLAGS ?= -O3
+
+CURL_CFLAGS := $(shell curl-config --cflags)
+CURL_LDFLAGS := $(shell curl-config --libs)
+
+PROGDIR := ../../programs
+LIBDIR := ../../lib
+ZSTD_CPPFLAGS := -I$(PROGDIR) -I$(LIBDIR) -I$(LIBDIR)/common
+
+REGRESSION_CFLAGS = $(CFLAGS) $(CURL_CFLAGS)
+REGRESSION_CPPFLAGS = $(CPPFLAGS) $(ZSTD_CPPFLAGS)
+REGRESSION_LDFLAGS = $(LDFLAGS) $(CURL_LDFLAGS)
+
+all: test
+
+xxhash.o: $(LIBDIR)/common/xxhash.c $(LIBDIR)/common/xxhash.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+util.o: $(PROGDIR)/util.c $(PROGDIR)/util.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+data.o: data.c data.h $(PROGDIR)/util.h $(LIBDIR)/common/xxhash.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+config.o: config.c config.h levels.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+method.h: data.h config.h result.h
+
+method.o: method.c method.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+result.o: result.c result.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+test.o: test.c data.h config.h method.h
+	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
+
+libzstd.a:
+	$(MAKE) -C $(LIBDIR) libzstd.a
+	cp $(LIBDIR)/libzstd.a .
+
+test: test.o data.o config.o util.o method.o result.o xxhash.o libzstd.a
+	$(CC) $^ $(REGRESSION_LDFLAGS) -o $@
+
+.PHONY: clean
+clean:
+	$(MAKE) -C $(LIBDIR) clean
+	$(RM) *.o *.a test

tests/regression/config.c

@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "config.h"
+
+/* Define a config for each fast level we want to test with. */
+#define FAST_LEVEL(x)                                               \
+    param_value_t const level_fast##x##_param_values[] = {          \
+        {.param = ZSTD_c_compressionLevel, .value = -x},            \
+    };                                                              \
+    config_t const level_fast##x = {                                \
+        .name = "level -" #x,                                       \
+        .cli_args = "--fast=" #x,                                   \
+        .param_values = PARAM_VALUES(level_fast##x##_param_values), \
+    };                                                              \
+    config_t const level_fast##x##_dict = {                         \
+        .name = "level -" #x " with dict",                          \
+        .cli_args = "--fast=" #x,                                   \
+        .param_values = PARAM_VALUES(level_fast##x##_param_values), \
+        .use_dictionary = 1,                                        \
+    };
+
+/* Define a config for each level we want to test with. */
+#define LEVEL(x)                                                \
+    param_value_t const level_##x##_param_values[] = {          \
+        {.param = ZSTD_c_compressionLevel, .value = x},         \
+    };                                                          \
+    config_t const level_##x = {                                \
+        .name = "level " #x,                                    \
+        .cli_args = "-" #x,                                     \
+        .param_values = PARAM_VALUES(level_##x##_param_values), \
+    };                                                          \
+    config_t const level_##x##_dict = {                         \
+        .name = "level " #x " with dict",                       \
+        .cli_args = "-" #x,                                     \
+        .param_values = PARAM_VALUES(level_##x##_param_values), \
+        .use_dictionary = 1,                                    \
+    };
+
+#define PARAM_VALUES(pv) \
+    { .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) }
+
+#include "levels.h"
+
+#undef LEVEL
+#undef FAST_LEVEL
+
+static config_t no_pledged_src_size = {
+    .name = "no source size",
+    .cli_args = "",
+    .param_values = PARAM_VALUES(level_0_param_values),
+    .no_pledged_src_size = 1,
+};
+
+static config_t const* g_configs[] = {
+
+#define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict,
+#define LEVEL(x) &level_##x, &level_##x##_dict,
+#include "levels.h"
+#undef LEVEL
+#undef FAST_LEVEL
+
+    &no_pledged_src_size,
+    NULL,
+};
+
+config_t const* const* configs = g_configs;
+
+int config_skip_data(config_t const* config, data_t const* data) {
+    return config->use_dictionary && !data_has_dict(data);
+}
+
+int config_get_level(config_t const* config) {
+    param_values_t const params = config->param_values;
+    size_t i;
+    for (size_t i = 0; i < params.size; ++i) {
+        if (params.data[i].param == ZSTD_c_compressionLevel)
+            return (int)params.data[i].value;
+    }
+    return CONFIG_NO_LEVEL;
+}
+
+ZSTD_parameters config_get_zstd_params(
+    config_t const* config,
+    uint64_t srcSize,
+    size_t dictSize) {
+    ZSTD_parameters zparams = {};
+    param_values_t const params = config->param_values;
+    int level = config_get_level(config);
+    if (level == CONFIG_NO_LEVEL)
+        level = 3;
+    zparams = ZSTD_getParams(
+        level,
+        config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : srcSize,
+        dictSize);
+    for (size_t i = 0; i < params.size; ++i) {
+        unsigned const value = params.data[i].value;
+        switch (params.data[i].param) {
+            case ZSTD_c_contentSizeFlag:
+                zparams.fParams.contentSizeFlag = value;
+                break;
+            case ZSTD_c_checksumFlag:
+                zparams.fParams.checksumFlag = value;
+                break;
+            case ZSTD_c_dictIDFlag:
+                zparams.fParams.noDictIDFlag = !value;
+                break;
+            case ZSTD_c_windowLog:
+                zparams.cParams.windowLog = value;
+                break;
+            case ZSTD_c_chainLog:
+                zparams.cParams.chainLog = value;
+                break;
+            case ZSTD_c_hashLog:
+                zparams.cParams.hashLog = value;
+                break;
+            case ZSTD_c_searchLog:
+                zparams.cParams.searchLog = value;
+                break;
+            case ZSTD_c_minMatch:
+                zparams.cParams.minMatch = value;
+                break;
+            case ZSTD_c_targetLength:
+                zparams.cParams.targetLength = value;
+                break;
+            case ZSTD_c_compressionStrategy:
+                zparams.cParams.strategy = (ZSTD_strategy)value;
+                break;
+            default:
+                break;
+        }
+    }
+    return zparams;
+}

tests/regression/config.h

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef CONFIG_H
+#define CONFIG_H
+
+#include <stddef.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h>
+
+#include "data.h"
+
+typedef struct {
+    ZSTD_cParameter param;
+    int value;
+} param_value_t;
+
+typedef struct {
+    size_t size;
+    param_value_t const* data;
+} param_values_t;
+
+/**
+ * The config tells the compression method what options to use.
+ */
+typedef struct {
+    const char* name;  /**< Identifies the config in the results table */
+    /**
+     * Optional arguments to pass to the CLI. If not set, CLI-based methods
+     * will skip this config.
+     */
+    char const* cli_args;
+    /**
+     * Parameters to pass to the advanced API. If the advanced API isn't used,
+     * the parameters will be derived from these.
+     */
+    param_values_t param_values;
+    /**
+     * Boolean parameter that says if we should use a dictionary. If the data
+     * doesn't have a dictionary, this config is skipped. Defaults to no.
+     */
+    int use_dictionary;
+    /**
+     * Boolean parameter that says if we should pass the pledged source size
+     * when the method allows it. Defaults to yes.
+     */
+    int no_pledged_src_size;
+} config_t;
+
+/**
+ * Returns true if the config should skip this data.
+ * For instance, if the config requires a dictionary but the data doesn't have
+ * one.
+ */
+int config_skip_data(config_t const* config, data_t const* data);
+
+#define CONFIG_NO_LEVEL (-ZSTD_TARGETLENGTH_MAX - 1)
+/**
+ * Returns the compression level specified by the config, or CONFIG_NO_LEVEL if
+ * no level is specified. Note that 0 is a valid compression level, meaning
+ * default.
+ */
+int config_get_level(config_t const* config);
+
+/**
+ * Returns the compression parameters specified by the config.
+ */
+ZSTD_parameters config_get_zstd_params(
+    config_t const* config,
+    uint64_t srcSize,
+    size_t dictSize);
+
+/**
+ * The NULL-terminated list of configs.
+ */
+extern config_t const* const* configs;
+
+#endif

tests/regression/data.c

@@ -0,0 +1,617 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "data.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <sys/stat.h>
+
+#include <curl/curl.h>
+
+#include "mem.h"
+#include "util.h"
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"
+
+/**
+ * Data objects
+ */
+
+#define REGRESSION_RELEASE(x) \
+    "https://github.com/facebook/zstd/releases/download/regression-data/" x
+
+data_t silesia = {
+    .name = "silesia",
+    .type = data_type_dir,
+    .data =
+        {
+            .url = REGRESSION_RELEASE("silesia.tar.zst"),
+            .xxhash64 = 0x48a199f92f93e977LL,
+        },
+};
+
+data_t silesia_tar = {
+    .name = "silesia.tar",
+    .type = data_type_file,
+    .data =
+        {
+            .url = REGRESSION_RELEASE("silesia.tar.zst"),
+            .xxhash64 = 0x48a199f92f93e977LL,
+        },
+};
+
+data_t github = {
+    .name = "github",
+    .type = data_type_dir,
+    .data =
+        {
+            .url = REGRESSION_RELEASE("github.tar.zst"),
+            .xxhash64 = 0xa9b1b44b020df292LL,
+        },
+    .dict =
+        {
+            .url = REGRESSION_RELEASE("github.dict.zst"),
+            .xxhash64 = 0x1eddc6f737d3cb53LL,
+
+        },
+};
+
+static data_t* g_data[] = {
+    &silesia,
+    &silesia_tar,
+    &github,
+    NULL,
+};
+
+data_t const* const* data = (data_t const* const*)g_data;
+
+/**
+ * data helpers.
+ */
+
+int data_has_dict(data_t const* data) {
+    return data->dict.url != NULL;
+}
+
+/**
+ * data buffer helper functions (documented in header).
+ */
+
+data_buffer_t data_buffer_create(size_t const capacity) {
+    data_buffer_t buffer = {};
+
+    buffer.data = (uint8_t*)malloc(capacity);
+    if (buffer.data == NULL)
+        return buffer;
+    buffer.capacity = capacity;
+    return buffer;
+}
+
+data_buffer_t data_buffer_read(char const* filename) {
+    data_buffer_t buffer = {};
+
+    uint64_t const size = UTIL_getFileSize(filename);
+    if (size == UTIL_FILESIZE_UNKNOWN) {
+        fprintf(stderr, "unknown size for %s\n", filename);
+        return buffer;
+    }
+
+    buffer.data = (uint8_t*)malloc(size);
+    if (buffer.data == NULL) {
+        fprintf(stderr, "malloc failed\n");
+        return buffer;
+    }
+    buffer.capacity = size;
+
+    FILE* file = fopen(filename, "rb");
+    if (file == NULL) {
+        fprintf(stderr, "file null\n");
+        goto err;
+    }
+    buffer.size = fread(buffer.data, 1, buffer.capacity, file);
+    fclose(file);
+    if (buffer.size != buffer.capacity) {
+        fprintf(stderr, "read %zu != %zu\n", buffer.size, buffer.capacity);
+        goto err;
+    }
+
+    return buffer;
+err:
+    free(buffer.data);
+    memset(&buffer, 0, sizeof(buffer));
+    return buffer;
+}
+
+data_buffer_t data_buffer_get_data(data_t const* data) {
+    data_buffer_t const kEmptyBuffer = {};
+
+    if (data->type != data_type_file)
+        return kEmptyBuffer;
+
+    return data_buffer_read(data->data.path);
+}
+
+data_buffer_t data_buffer_get_dict(data_t const* data) {
+    data_buffer_t const kEmptyBuffer = {};
+
+    if (!data_has_dict(data))
+        return kEmptyBuffer;
+
+    return data_buffer_read(data->dict.path);
+}
+
+int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2) {
+    size_t const size =
+        buffer1.size < buffer2.size ? buffer1.size : buffer2.size;
+    int const cmp = memcmp(buffer1.data, buffer2.data, size);
+    if (cmp != 0)
+        return cmp;
+    if (buffer1.size < buffer2.size)
+        return -1;
+    if (buffer1.size == buffer2.size)
+        return 0;
+    assert(buffer1.size > buffer2.size);
+    return 1;
+}
+
+void data_buffer_free(data_buffer_t buffer) {
+    free(buffer.data);
+}
+
+/**
+ * data filenames helpers.
+ */
+
+data_filenames_t data_filenames_get(data_t const* data) {
+    data_filenames_t filenames = {.buffer = NULL, .size = 0};
+    char const* path = data->data.path;
+
+    filenames.filenames = UTIL_createFileList(
+        &path,
+        1,
+        &filenames.buffer,
+        &filenames.size,
+        /* followLinks */ 0);
+    return filenames;
+}
+
+void data_filenames_free(data_filenames_t filenames) {
+    UTIL_freeFileList(filenames.filenames, filenames.buffer);
+}
+
+/**
+ * data buffers helpers.
+ */
+
+data_buffers_t data_buffers_get(data_t const* data) {
+    data_buffers_t buffers = {.size = 0};
+    data_filenames_t filenames = data_filenames_get(data);
+    if (filenames.size == 0)
+        return buffers;
+
+    data_buffer_t* buffersPtr =
+        (data_buffer_t*)malloc(filenames.size * sizeof(data_buffer_t));
+    if (buffersPtr == NULL)
+        return buffers;
+    buffers.buffers = (data_buffer_t const*)buffersPtr;
+    buffers.size = filenames.size;
+
+    for (size_t i = 0; i < filenames.size; ++i) {
+        buffersPtr[i] = data_buffer_read(filenames.filenames[i]);
+        if (buffersPtr[i].data == NULL) {
+            data_buffers_t const kEmptyBuffer = {};
+            data_buffers_free(buffers);
+            return kEmptyBuffer;
+        }
+    }
+
+    return buffers;
+}
+
+/**
+ * Frees the data buffers.
+ */
+void data_buffers_free(data_buffers_t buffers) {
+    free((data_buffer_t*)buffers.buffers);
+}
+
+/**
+ * Initialization and download functions.
+ */
+
+static char* g_data_dir = NULL;
+
+/* mkdir -p */
+static int ensure_directory_exists(char const* indir) {
+    char* const dir = strdup(indir);
+    char* end = dir;
+    int ret = 0;
+    if (dir == NULL) {
+        ret = EINVAL;
+        goto out;
+    }
+    do {
+        /* Find the next directory level. */
+        for (++end; *end != '\0' && *end != '/'; ++end)
+            ;
+        /* End the string there, make the directory, and restore the string. */
+        char const save = *end;
+        *end = '\0';
+        int const isdir = UTIL_isDirectory(dir);
+        ret = mkdir(dir, S_IRWXU);
+        *end = save;
+        /* Its okay if the directory already exists. */
+        if (ret == 0 || (errno == EEXIST && isdir))
+            continue;
+        ret = errno;
+        fprintf(stderr, "mkdir() failed\n");
+        goto out;
+    } while (*end != '\0');
+
+    ret = 0;
+out:
+    free(dir);
+    return ret;
+}
+
+/** Concatenate 3 strings into a new buffer. */
+static char* cat3(char const* str1, char const* str2, char const* str3) {
+    size_t const size1 = strlen(str1);
+    size_t const size2 = strlen(str2);
+    size_t const size3 = str3 == NULL ? 0 : strlen(str3);
+    size_t const size = size1 + size2 + size3 + 1;
+    char* const dst = (char*)malloc(size);
+    if (dst == NULL)
+        return NULL;
+    strcpy(dst, str1);
+    strcpy(dst + size1, str2);
+    if (str3 != NULL)
+        strcpy(dst + size1 + size2, str3);
+    assert(strlen(dst) == size1 + size2 + size3);
+    return dst;
+}
+
+static char* cat2(char const* str1, char const* str2) {
+    return cat3(str1, str2, NULL);
+}
+
+/**
+ * State needed by the curl callback.
+ * It takes data from curl, hashes it, and writes it to the file.
+ */
+typedef struct {
+    FILE* file;
+    XXH64_state_t xxhash64;
+    int error;
+} curl_data_t;
+
+/** Create the curl state. */
+static curl_data_t curl_data_create(
+    data_resource_t const* resource,
+    data_type_t type) {
+    curl_data_t cdata = {};
+
+    XXH64_reset(&cdata.xxhash64, 0);
+
+    assert(UTIL_isDirectory(g_data_dir));
+
+    if (type == data_type_file) {
+        /* Decompress the resource and store to the path. */
+        char* cmd = cat3("zstd -dqfo '", resource->path, "'");
+        if (cmd == NULL) {
+            cdata.error = ENOMEM;
+            return cdata;
+        }
+        cdata.file = popen(cmd, "w");
+        free(cmd);
+    } else {
+        /* Decompress and extract the resource to the cache directory. */
+        char* cmd = cat3("zstd -dc | tar -x -C '", g_data_dir, "'");
+        if (cmd == NULL) {
+            cdata.error = ENOMEM;
+            return cdata;
+        }
+        cdata.file = popen(cmd, "w");
+        free(cmd);
+    }
+    if (cdata.file == NULL) {
+        cdata.error = errno;
+    }
+
+    return cdata;
+}
+
+/** Free the curl state. */
+static int curl_data_free(curl_data_t cdata) {
+    return pclose(cdata.file);
+}
+
+/** curl callback. Updates the hash, and writes to the file. */
+static size_t curl_write(void* data, size_t size, size_t count, void* ptr) {
+    curl_data_t* cdata = (curl_data_t*)ptr;
+    size_t const written = fwrite(data, size, count, cdata->file);
+    XXH64_update(&cdata->xxhash64, data, written * size);
+    return written;
+}
+
+static int curl_download_resource(
+    CURL* curl,
+    data_resource_t const* resource,
+    data_type_t type) {
+    curl_data_t cdata;
+    /* Download the data. */
+    if (curl_easy_setopt(curl, CURLOPT_URL, resource->url) != 0)
+        return EINVAL;
+    if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cdata) != 0)
+        return EINVAL;
+    cdata = curl_data_create(resource, type);
+    if (cdata.error != 0)
+        return cdata.error;
+    int const curl_err = curl_easy_perform(curl);
+    int const close_err = curl_data_free(cdata);
+    if (curl_err) {
+        fprintf(
+            stderr,
+            "downloading '%s' for '%s' failed\n",
+            resource->url,
+            resource->path);
+        return EIO;
+    }
+    if (close_err) {
+        fprintf(stderr, "writing data to '%s' failed\n", resource->path);
+        return EIO;
+    }
+    /* check that the file exists. */
+    if (type == data_type_file && !UTIL_isRegularFile(resource->path)) {
+        fprintf(stderr, "output file '%s' does not exist\n", resource->path);
+        return EIO;
+    }
+    if (type == data_type_dir && !UTIL_isDirectory(resource->path)) {
+        fprintf(
+            stderr, "output directory '%s' does not exist\n", resource->path);
+        return EIO;
+    }
+    /* Check that the hash matches. */
+    if (XXH64_digest(&cdata.xxhash64) != resource->xxhash64) {
+        fprintf(
+            stderr,
+            "checksum does not match: 0x%llxLL != 0x%llxLL\n",
+            (unsigned long long)XXH64_digest(&cdata.xxhash64),
+            (unsigned long long)resource->xxhash64);
+        return EINVAL;
+    }
+
+    return 0;
+}
+
+/** Download a single data object. */
+static int curl_download_datum(CURL* curl, data_t const* data) {
+    int ret;
+    ret = curl_download_resource(curl, &data->data, data->type);
+    if (ret != 0)
+        return ret;
+    if (data_has_dict(data)) {
+        ret = curl_download_resource(curl, &data->dict, data_type_file);
+        if (ret != 0)
+            return ret;
+    }
+    return ret;
+}
+
+/** Download all the data. */
+static int curl_download_data(data_t const* const* data) {
+    if (curl_global_init(CURL_GLOBAL_ALL) != 0)
+        return EFAULT;
+
+    curl_data_t cdata = {};
+    CURL* curl = curl_easy_init();
+    int err = EFAULT;
+
+    if (curl == NULL)
+        return EFAULT;
+
+    if (curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L) != 0)
+        goto out;
+    if (curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L) != 0)
+        goto out;
+    if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write) != 0)
+        goto out;
+
+    assert(data != NULL);
+    for (; *data != NULL; ++data) {
+        if (curl_download_datum(curl, *data) != 0)
+            goto out;
+    }
+
+    err = 0;
+out:
+    curl_easy_cleanup(curl);
+    curl_global_cleanup();
+    return err;
+}
+
+/** Fill the path member variable of the data objects. */
+static int data_create_paths(data_t* const* data, char const* dir) {
+    size_t const dirlen = strlen(dir);
+    assert(data != NULL);
+    for (; *data != NULL; ++data) {
+        data_t* const datum = *data;
+        datum->data.path = cat3(dir, "/", datum->name);
+        if (datum->data.path == NULL)
+            return ENOMEM;
+        if (data_has_dict(datum)) {
+            datum->dict.path = cat2(datum->data.path, ".dict");
+            if (datum->dict.path == NULL)
+                return ENOMEM;
+        }
+    }
+    return 0;
+}
+
+/** Free the path member variable of the data objects. */
+static void data_free_paths(data_t* const* data) {
+    assert(data != NULL);
+    for (; *data != NULL; ++data) {
+        data_t* datum = *data;
+        free((void*)datum->data.path);
+        free((void*)datum->dict.path);
+        datum->data.path = NULL;
+        datum->dict.path = NULL;
+    }
+}
+
+static char const kStampName[] = "STAMP";
+
+static void xxh_update_le(XXH64_state_t* state, uint64_t data) {
+    if (!MEM_isLittleEndian())
+        data = MEM_swap64(data);
+    XXH64_update(state, &data, sizeof(data));
+}
+
+/** Hash the data to create the stamp. */
+static uint64_t stamp_hash(data_t const* const* data) {
+    XXH64_state_t state;
+
+    XXH64_reset(&state, 0);
+    assert(data != NULL);
+    for (; *data != NULL; ++data) {
+        data_t const* datum = *data;
+        /* We don't care about the URL that we fetch from. */
+        /* The path is derived from the name. */
+        XXH64_update(&state, datum->name, strlen(datum->name));
+        xxh_update_le(&state, datum->data.xxhash64);
+        xxh_update_le(&state, datum->dict.xxhash64);
+        xxh_update_le(&state, datum->type);
+    }
+    return XXH64_digest(&state);
+}
+
+/** Check if the stamp matches the stamp in the cache directory. */
+static int stamp_check(char const* dir, data_t const* const* data) {
+    char* stamp = cat3(dir, "/", kStampName);
+    uint64_t const expected = stamp_hash(data);
+    XXH64_canonical_t actual;
+    FILE* stampfile = NULL;
+    int matches = 0;
+
+    if (stamp == NULL)
+        goto out;
+    if (!UTIL_isRegularFile(stamp)) {
+        fprintf(stderr, "stamp does not exist: recreating the data cache\n");
+        goto out;
+    }
+
+    stampfile = fopen(stamp, "rb");
+    if (stampfile == NULL) {
+        fprintf(stderr, "could not open stamp: recreating the data cache\n");
+        goto out;
+    }
+
+    size_t b;
+    if ((b = fread(&actual, sizeof(actual), 1, stampfile)) != 1) {
+        fprintf(stderr, "invalid stamp: recreating the data cache\n");
+        goto out;
+    }
+
+    matches = (expected == XXH64_hashFromCanonical(&actual));
+    if (matches)
+        fprintf(stderr, "stamp matches: reusing the cached data\n");
+    else
+        fprintf(stderr, "stamp does not match: recreating the data cache\n");
+
+out:
+    free(stamp);
+    if (stampfile != NULL)
+        fclose(stampfile);
+    return matches;
+}
+
+/** On success write a new stamp, on failure delete the old stamp. */
+static int
+stamp_write(char const* dir, data_t const* const* data, int const data_err) {
+    char* stamp = cat3(dir, "/", kStampName);
+    FILE* stampfile = NULL;
+    int err = EIO;
+
+    if (stamp == NULL)
+        return ENOMEM;
+
+    if (data_err != 0) {
+        err = data_err;
+        goto out;
+    }
+    XXH64_canonical_t hash;
+
+    XXH64_canonicalFromHash(&hash, stamp_hash(data));
+
+    stampfile = fopen(stamp, "wb");
+    if (stampfile == NULL)
+        goto out;
+    if (fwrite(&hash, sizeof(hash), 1, stampfile) != 1)
+        goto out;
+    err = 0;
+    fprintf(stderr, "stamped new data cache\n");
+out:
+    if (err != 0)
+        /* Ignore errors. */
+        unlink(stamp);
+    free(stamp);
+    if (stampfile != NULL)
+        fclose(stampfile);
+    return err;
+}
+
+int data_init(char const* dir) {
+    int err;
+
+    if (dir == NULL)
+        return EINVAL;
+
+    /* This must be first to simplify logic. */
+    err = ensure_directory_exists(dir);
+    if (err != 0)
+        return err;
+
+    /* Save the cache directory. */
+    g_data_dir = strdup(dir);
+    if (g_data_dir == NULL)
+        return ENOMEM;
+
+    err = data_create_paths(g_data, dir);
+    if (err != 0)
+        return err;
+
+    /* If the stamp matches then we are good to go.
+     * This must be called before any modifications to the data cache.
+     * After this point, we MUST call stamp_write() to update the STAMP,
+     * since we've updated the data cache.
+     */
+    if (stamp_check(dir, data))
+        return 0;
+
+    err = curl_download_data(data);
+    if (err != 0)
+        goto out;
+
+out:
+    /* This must be last, since it must know if data_init() succeeded. */
+    stamp_write(dir, data, err);
+    return err;
+}
+
+void data_finish(void) {
+    data_free_paths(g_data);
+    free(g_data_dir);
+    g_data_dir = NULL;
+}

tests/regression/data.h

@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef DATA_H
+#define DATA_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef enum {
+    data_type_file = 1,  /**< This data is a file. *.zst */
+    data_type_dir = 2,   /**< This data is a directory. *.tar.zst */
+} data_type_t;
+
+typedef struct {
+    char const* url;   /**< Where to get this resource. */
+    uint64_t xxhash64; /**< Hash of the url contents. */
+    char const* path;  /**< The path of the unpacked resource (derived). */
+} data_resource_t;
+
+typedef struct {
+    data_resource_t data;
+    data_resource_t dict;
+    data_type_t type;  /**< The type of the data. */
+    char const* name;  /**< The logical name of the data (no extension). */
+} data_t;
+
+/**
+ * The NULL-terminated list of data objects.
+ */
+extern data_t const* const* data;
+
+
+int data_has_dict(data_t const* data);
+
+/**
+ * Initializes the data module and downloads the data necessary.
+ * Caches the downloads in dir. We add a stamp file in the directory after
+ * a successful download. If a stamp file already exists, and matches our
+ * current data stamp, we will use the cached data without downloading.
+ *
+ * @param dir The directory to cache the downloaded data into.
+ *
+ * @returns 0 on success.
+ */
+int data_init(char const* dir);
+
+/**
+ * Must be called at exit to free resources allocated by data_init().
+ */
+void data_finish(void);
+
+typedef struct {
+    uint8_t* data;
+    size_t size;
+    size_t capacity;
+} data_buffer_t;
+
+/**
+ * Read the file that data points to into a buffer.
+ * NOTE: data must be a file, not a directory.
+ *
+ * @returns The buffer, which is NULL on failure.
+ */
+data_buffer_t data_buffer_get_data(data_t const* data);
+
+/**
+ * Read the dictionary that the data points to into a buffer.
+ *
+ * @returns The buffer, which is NULL on failure.
+ */
+data_buffer_t data_buffer_get_dict(data_t const* data);
+
+/**
+ * Read the contents of filename into a buffer.
+ *
+ * @returns The buffer, which is NULL on failure.
+ */
+data_buffer_t data_buffer_read(char const* filename);
+
+/**
+ * Create a buffer with the specified capacity.
+ *
+ * @returns The buffer, which is NULL on failure.
+ */
+data_buffer_t data_buffer_create(size_t capacity);
+
+/**
+ * Calls memcmp() on the contents [0, size) of both buffers.
+ */
+int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2);
+
+/**
+ * Frees an allocated buffer.
+ */
+void data_buffer_free(data_buffer_t buffer);
+
+typedef struct {
+    char* buffer;
+    char const** filenames;
+    unsigned size;
+} data_filenames_t;
+
+/**
+ * Get a recursive list of filenames in the data object. If it is a file, it
+ * will only contain one entry. If it is a directory, it will recursively walk
+ * the directory.
+ *
+ * @returns The list of filenames, which has size 0 and NULL pointers on error.
+ */
+data_filenames_t data_filenames_get(data_t const* data);
+
+/**
+ * Frees the filenames table.
+ */
+void data_filenames_free(data_filenames_t filenames);
+
+typedef struct {
+    data_buffer_t const* buffers;
+    size_t size;
+} data_buffers_t;
+
+/**
+ * @returns a list of buffers for every file in data. It is zero sized on error.
+ */
+data_buffers_t data_buffers_get(data_t const* data);
+
+/**
+ * Frees the data buffers.
+ */
+void data_buffers_free(data_buffers_t buffers);
+
+#endif

tests/regression/levels.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef LEVEL
+# error LEVEL(x) must be defined
+#endif
+#ifndef FAST_LEVEL
+# error FAST_LEVEL(x) must be defined
+#endif
+
+/**
+ * The levels are chosen to trigger every strategy in every source size,
+ * as well as some fast levels and the default level.
+ * If you change the compression levels, you should probably update these.
+ */
+
+FAST_LEVEL(5)
+
+FAST_LEVEL(3)
+
+FAST_LEVEL(1)
+LEVEL(0)
+LEVEL(1)
+
+LEVEL(3)
+LEVEL(4)
+LEVEL(5)
+LEVEL(6)
+LEVEL(7)
+
+LEVEL(9)
+
+LEVEL(13)
+
+LEVEL(16)
+
+LEVEL(19)

tests/regression/method.c

@@ -0,0 +1,554 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "method.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <zstd.h>
+
+#define MIN(x, y) ((x) < (y) ? (x) : (y))
+
+static char const* g_zstdcli = NULL;
+
+void method_set_zstdcli(char const* zstdcli) {
+    g_zstdcli = zstdcli;
+}
+
+/**
+ * Macro to get a pointer of type, given ptr, which is a member variable with
+ * the given name, member.
+ *
+ *     method_state_t* base = ...;
+ *     buffer_state_t* state = container_of(base, buffer_state_t, base);
+ */
+#define container_of(ptr, type, member) \
+    ((type*)(ptr == NULL ? NULL : (char*)(ptr)-offsetof(type, member)))
+
+/** State to reuse the same buffers between compression calls. */
+typedef struct {
+    method_state_t base;
+    data_buffers_t inputs; /**< The input buffer for each file. */
+    data_buffer_t dictionary; /**< The dictionary. */
+    data_buffer_t compressed; /**< The compressed data buffer. */
+    data_buffer_t decompressed; /**< The decompressed data buffer. */
+} buffer_state_t;
+
+static size_t buffers_max_size(data_buffers_t buffers) {
+    size_t max = 0;
+    for (size_t i = 0; i < buffers.size; ++i) {
+        if (buffers.buffers[i].size > max)
+            max = buffers.buffers[i].size;
+    }
+    return max;
+}
+
+static method_state_t* buffer_state_create(data_t const* data) {
+    buffer_state_t* state = (buffer_state_t*)calloc(1, sizeof(buffer_state_t));
+    if (state == NULL)
+        return NULL;
+    state->base.data = data;
+    state->inputs = data_buffers_get(data);
+    state->dictionary = data_buffer_get_dict(data);
+    size_t const max_size = buffers_max_size(state->inputs);
+    state->compressed = data_buffer_create(ZSTD_compressBound(max_size));
+    state->decompressed = data_buffer_create(max_size);
+    return &state->base;
+}
+
+static void buffer_state_destroy(method_state_t* base) {
+    if (base == NULL)
+        return;
+    buffer_state_t* state = container_of(base, buffer_state_t, base);
+    free(state);
+}
+
+static int buffer_state_bad(
+    buffer_state_t const* state,
+    config_t const* config) {
+    if (state == NULL) {
+        fprintf(stderr, "buffer_state_t is NULL\n");
+        return 1;
+    }
+    if (state->inputs.size == 0 || state->compressed.data == NULL ||
+        state->decompressed.data == NULL) {
+        fprintf(stderr, "buffer state allocation failure\n");
+        return 1;
+    }
+    if (config->use_dictionary && state->dictionary.data == NULL) {
+        fprintf(stderr, "dictionary loading failed\n");
+        return 1;
+    }
+    return 0;
+}
+
+static result_t simple_compress(method_state_t* base, config_t const* config) {
+    buffer_state_t* state = container_of(base, buffer_state_t, base);
+
+    if (buffer_state_bad(state, config))
+        return result_error(result_error_system_error);
+
+    /* Keep the tests short by skipping directories, since behavior shouldn't
+     * change.
+     */
+    if (base->data->type != data_type_file)
+        return result_error(result_error_skip);
+
+    if (config->use_dictionary || config->no_pledged_src_size)
+        return result_error(result_error_skip);
+
+    /* If the config doesn't specify a level, skip. */
+    int const level = config_get_level(config);
+    if (level == CONFIG_NO_LEVEL)
+        return result_error(result_error_skip);
+
+    data_buffer_t const input = state->inputs.buffers[0];
+
+    /* Compress, decompress, and check the result. */
+    state->compressed.size = ZSTD_compress(
+        state->compressed.data,
+        state->compressed.capacity,
+        input.data,
+        input.size,
+        level);
+    if (ZSTD_isError(state->compressed.size))
+        return result_error(result_error_compression_error);
+
+    state->decompressed.size = ZSTD_decompress(
+        state->decompressed.data,
+        state->decompressed.capacity,
+        state->compressed.data,
+        state->compressed.size);
+    if (ZSTD_isError(state->decompressed.size))
+        return result_error(result_error_decompression_error);
+    if (data_buffer_compare(input, state->decompressed))
+        return result_error(result_error_round_trip_error);
+
+    result_data_t data;
+    data.total_size = state->compressed.size;
+    return result_data(data);
+}
+
+static result_t compress_cctx_compress(
+    method_state_t* base,
+    config_t const* config) {
+    buffer_state_t* state = container_of(base, buffer_state_t, base);
+
+    if (buffer_state_bad(state, config))
+        return result_error(result_error_system_error);
+
+    if (config->no_pledged_src_size)
+        return result_error(result_error_skip);
+
+    if (base->data->type != data_type_dir)
+        return result_error(result_error_skip);
+
+    int const level = config_get_level(config);
+    if (level == CONFIG_NO_LEVEL)
+        return result_error(result_error_skip);
+
+    ZSTD_CCtx* cctx = ZSTD_createCCtx();
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
+    if (cctx == NULL || dctx == NULL) {
+        fprintf(stderr, "context creation failed\n");
+        return result_error(result_error_system_error);
+    }
+
+    result_t result;
+    result_data_t data = {.total_size = 0};
+    for (size_t i = 0; i < state->inputs.size; ++i) {
+        data_buffer_t const input = state->inputs.buffers[i];
+
+        if (config->use_dictionary)
+            state->compressed.size = ZSTD_compress_usingDict(
+                cctx,
+                state->compressed.data,
+                state->compressed.capacity,
+                input.data,
+                input.size,
+                state->dictionary.data,
+                state->dictionary.size,
+                level);
+        else
+            state->compressed.size = ZSTD_compressCCtx(
+                cctx,
+                state->compressed.data,
+                state->compressed.capacity,
+                input.data,
+                input.size,
+                level);
+        if (ZSTD_isError(state->compressed.size)) {
+            result = result_error(result_error_compression_error);
+            goto out;
+        }
+
+        if (config->use_dictionary)
+            state->decompressed.size = ZSTD_decompress_usingDict(
+                dctx,
+                state->decompressed.data,
+                state->decompressed.capacity,
+                state->compressed.data,
+                state->compressed.size,
+                state->dictionary.data,
+                state->dictionary.size);
+        else
+            state->decompressed.size = ZSTD_decompressDCtx(
+                dctx,
+                state->decompressed.data,
+                state->decompressed.capacity,
+                state->compressed.data,
+                state->compressed.size);
+        if (ZSTD_isError(state->decompressed.size)) {
+            result = result_error(result_error_decompression_error);
+            goto out;
+        }
+        if (data_buffer_compare(input, state->decompressed)) {
+            result = result_error(result_error_round_trip_error);
+            goto out;
+        }
+
+        data.total_size += state->compressed.size;
+    }
+
+    result = result_data(data);
+out:
+    ZSTD_freeCCtx(cctx);
+    ZSTD_freeDCtx(dctx);
+    return result;
+}
+
+/** Generic state creation function. */
+static method_state_t* method_state_create(data_t const* data) {
+    method_state_t* state = (method_state_t*)malloc(sizeof(method_state_t));
+    if (state == NULL)
+        return NULL;
+    state->data = data;
+    return state;
+}
+
+static void method_state_destroy(method_state_t* state) {
+    free(state);
+}
+
+static result_t cli_compress(method_state_t* state, config_t const* config) {
+    if (config->cli_args == NULL)
+        return result_error(result_error_skip);
+
+    /* We don't support no pledged source size with directories. Too slow. */
+    if (state->data->type == data_type_dir && config->no_pledged_src_size)
+        return result_error(result_error_skip);
+
+    if (g_zstdcli == NULL)
+        return result_error(result_error_system_error);
+
+    /* '<zstd>' -cqr <args> [-D '<dict>'] '<file/dir>' */
+    char cmd[1024];
+    size_t const cmd_size = snprintf(
+        cmd,
+        sizeof(cmd),
+        "'%s' -cqr %s %s%s%s %s '%s'",
+        g_zstdcli,
+        config->cli_args,
+        config->use_dictionary ? "-D '" : "",
+        config->use_dictionary ? state->data->dict.path : "",
+        config->use_dictionary ? "'" : "",
+        config->no_pledged_src_size ? "<" : "",
+        state->data->data.path);
+    if (cmd_size >= sizeof(cmd)) {
+        fprintf(stderr, "command too large: %s\n", cmd);
+        return result_error(result_error_system_error);
+    }
+    FILE* zstd = popen(cmd, "r");
+    if (zstd == NULL) {
+        fprintf(stderr, "failed to popen command: %s\n", cmd);
+        return result_error(result_error_system_error);
+    }
+
+    char out[4096];
+    size_t total_size = 0;
+    while (1) {
+        size_t const size = fread(out, 1, sizeof(out), zstd);
+        total_size += size;
+        if (size != sizeof(out))
+            break;
+    }
+    if (ferror(zstd) || pclose(zstd) != 0) {
+        fprintf(stderr, "zstd failed with command: %s\n", cmd);
+        return result_error(result_error_compression_error);
+    }
+
+    result_data_t const data = {.total_size = total_size};
+    return result_data(data);
+}
+
+static int advanced_config(
+    ZSTD_CCtx* cctx,
+    buffer_state_t* state,
+    config_t const* config) {
+    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+    for (size_t p = 0; p < config->param_values.size; ++p) {
+        param_value_t const pv = config->param_values.data[p];
+        if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, pv.param, pv.value))) {
+            return 1;
+        }
+    }
+    if (config->use_dictionary) {
+        if (ZSTD_isError(ZSTD_CCtx_loadDictionary(
+                cctx, state->dictionary.data, state->dictionary.size))) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+static result_t advanced_one_pass_compress_output_adjustment(
+    method_state_t* base,
+    config_t const* config,
+    size_t const subtract) {
+    buffer_state_t* state = container_of(base, buffer_state_t, base);
+
+    if (buffer_state_bad(state, config))
+        return result_error(result_error_system_error);
+
+    ZSTD_CCtx* cctx = ZSTD_createCCtx();
+    result_t result;
+
+    if (!cctx || advanced_config(cctx, state, config)) {
+        result = result_error(result_error_compression_error);
+        goto out;
+    }
+
+    result_data_t data = {.total_size = 0};
+    for (size_t i = 0; i < state->inputs.size; ++i) {
+        data_buffer_t const input = state->inputs.buffers[i];
+
+        if (!config->no_pledged_src_size) {
+            if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) {
+                result = result_error(result_error_compression_error);
+                goto out;
+            }
+        }
+        size_t const size = ZSTD_compress2(
+            cctx,
+            state->compressed.data,
+            ZSTD_compressBound(input.size) - subtract,
+            input.data,
+            input.size);
+        if (ZSTD_isError(size)) {
+            result = result_error(result_error_compression_error);
+            goto out;
+        }
+        data.total_size += size;
+    }
+
+    result = result_data(data);
+out:
+    ZSTD_freeCCtx(cctx);
+    return result;
+}
+
+static result_t advanced_one_pass_compress(
+    method_state_t* base,
+    config_t const* config) {
+  return advanced_one_pass_compress_output_adjustment(base, config, 0);
+}
+
+static result_t advanced_one_pass_compress_small_output(
+    method_state_t* base,
+    config_t const* config) {
+  return advanced_one_pass_compress_output_adjustment(base, config, 1);
+}
+
+static result_t advanced_streaming_compress(
+    method_state_t* base,
+    config_t const* config) {
+    buffer_state_t* state = container_of(base, buffer_state_t, base);
+
+    if (buffer_state_bad(state, config))
+        return result_error(result_error_system_error);
+
+    ZSTD_CCtx* cctx = ZSTD_createCCtx();
+    result_t result;
+
+    if (!cctx || advanced_config(cctx, state, config)) {
+        result = result_error(result_error_compression_error);
+        goto out;
+    }
+
+    result_data_t data = {.total_size = 0};
+    for (size_t i = 0; i < state->inputs.size; ++i) {
+        data_buffer_t input = state->inputs.buffers[i];
+
+        if (!config->no_pledged_src_size) {
+            if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) {
+                result = result_error(result_error_compression_error);
+                goto out;
+            }
+        }
+
+        while (input.size > 0) {
+            ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
+            input.data += in.size;
+            input.size -= in.size;
+            ZSTD_EndDirective const op =
+                input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
+            size_t ret = 0;
+            while (in.pos < in.size || (op == ZSTD_e_end && ret != 0)) {
+                ZSTD_outBuffer out = {state->compressed.data,
+                                      MIN(state->compressed.capacity, 1024)};
+                ret = ZSTD_compressStream2(cctx, &out, &in, op);
+                if (ZSTD_isError(ret)) {
+                    result = result_error(result_error_compression_error);
+                    goto out;
+                }
+                data.total_size += out.pos;
+            }
+        }
+    }
+
+    result = result_data(data);
+out:
+    ZSTD_freeCCtx(cctx);
+    return result;
+}
+
+static result_t old_streaming_compress(
+    method_state_t* base,
+    config_t const* config) {
+    buffer_state_t* state = container_of(base, buffer_state_t, base);
+
+    if (buffer_state_bad(state, config))
+        return result_error(result_error_system_error);
+
+    int const level = config_get_level(config);
+    if (level == CONFIG_NO_LEVEL)
+        return result_error(result_error_skip);
+
+    ZSTD_CStream* zcs = ZSTD_createCStream();
+    result_t result;
+    if (zcs == NULL) {
+        result = result_error(result_error_compression_error);
+        goto out;
+    }
+    size_t zret;
+    if (config->use_dictionary) {
+        zret = ZSTD_initCStream_usingDict(
+            zcs, state->dictionary.data, state->dictionary.size, level);
+    } else {
+        zret = ZSTD_initCStream(zcs, level);
+    }
+    if (ZSTD_isError(zret)) {
+        result = result_error(result_error_compression_error);
+        goto out;
+    }
+
+    result_data_t data = {.total_size = 0};
+    for (size_t i = 0; i < state->inputs.size; ++i) {
+        data_buffer_t input = state->inputs.buffers[i];
+        zret = ZSTD_resetCStream(
+            zcs,
+            config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN
+                                        : input.size);
+        if (ZSTD_isError(zret)) {
+            result = result_error(result_error_compression_error);
+            goto out;
+        }
+
+        while (input.size > 0) {
+            ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)};
+            input.data += in.size;
+            input.size -= in.size;
+            ZSTD_EndDirective const op =
+                input.size > 0 ? ZSTD_e_continue : ZSTD_e_end;
+            zret = 0;
+            while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) {
+                ZSTD_outBuffer out = {state->compressed.data,
+                                      MIN(state->compressed.capacity, 1024)};
+                if (op == ZSTD_e_continue || in.pos < in.size)
+                    zret = ZSTD_compressStream(zcs, &out, &in);
+                else
+                    zret = ZSTD_endStream(zcs, &out);
+                if (ZSTD_isError(zret)) {
+                    result = result_error(result_error_compression_error);
+                    goto out;
+                }
+                data.total_size += out.pos;
+            }
+        }
+    }
+
+    result = result_data(data);
+out:
+    ZSTD_freeCStream(zcs);
+    return result;
+}
+
+method_t const simple = {
+    .name = "compress simple",
+    .create = buffer_state_create,
+    .compress = simple_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const compress_cctx = {
+    .name = "compress cctx",
+    .create = buffer_state_create,
+    .compress = compress_cctx_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const advanced_one_pass = {
+    .name = "advanced one pass",
+    .create = buffer_state_create,
+    .compress = advanced_one_pass_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const advanced_one_pass_small_out = {
+    .name = "advanced one pass small out",
+    .create = buffer_state_create,
+    .compress = advanced_one_pass_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const advanced_streaming = {
+    .name = "advanced streaming",
+    .create = buffer_state_create,
+    .compress = advanced_streaming_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const old_streaming = {
+    .name = "old streaming",
+    .create = buffer_state_create,
+    .compress = old_streaming_compress,
+    .destroy = buffer_state_destroy,
+};
+
+method_t const cli = {
+    .name = "zstdcli",
+    .create = method_state_create,
+    .compress = cli_compress,
+    .destroy = method_state_destroy,
+};
+
+static method_t const* g_methods[] = {
+    &simple,
+    &compress_cctx,
+    &cli,
+    &advanced_one_pass,
+    &advanced_one_pass_small_out,
+    &advanced_streaming,
+    &old_streaming,
+    NULL,
+};
+
+method_t const* const* methods = g_methods;

tests/regression/method.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef METHOD_H
+#define METHOD_H
+
+#include <stddef.h>
+
+#include "data.h"
+#include "config.h"
+#include "result.h"
+
+/**
+ * The base class for state that methods keep.
+ * All derived method state classes must have a member of this type.
+ */
+typedef struct {
+    data_t const* data;
+} method_state_t;
+
+/**
+ * A method that compresses the data using config.
+ */
+typedef struct {
+    char const* name;  /**< The identifier for this method in the results. */
+    /**
+     * Creates a state that must contain a member variable of method_state_t,
+     * and returns a pointer to that member variable.
+     *
+     * This method can be used to do expensive work that only depends on the
+     * data, like loading the data file into a buffer.
+     */
+    method_state_t* (*create)(data_t const* data);
+    /**
+     * Compresses the data in the state using the given config.
+     *
+     * @param state A pointer to the state returned by create().
+     *
+     * @returns The total compressed size on success, or an error code.
+     */
+    result_t (*compress)(method_state_t* state, config_t const* config);
+    /**
+     * Frees the state.
+     */
+    void (*destroy)(method_state_t* state);
+} method_t;
+
+/**
+ * Set the zstd cli path. Must be called before any methods are used.
+ */
+void method_set_zstdcli(char const* zstdcli);
+
+/**
+ * A NULL-terminated list of methods.
+ */
+extern method_t const* const* methods;
+
+#endif

tests/regression/result.c

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include "result.h"
+
+char const* result_get_error_string(result_t result) {
+    switch (result_get_error(result)) {
+        case result_error_ok:
+            return "okay";
+        case result_error_skip:
+            return "skip";
+        case result_error_system_error:
+            return "system error";
+        case result_error_compression_error:
+            return "compression error";
+        case result_error_decompression_error:
+            return "decompression error";
+        case result_error_round_trip_error:
+            return "round trip error";
+    }
+}

tests/regression/result.h

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef RESULT_H
+#define RESULT_H
+
+#include <stddef.h>
+
+/**
+ * The error type enum.
+ */
+typedef enum {
+    result_error_ok,                   /**< No error. */
+    result_error_skip,                 /**< This method was skipped. */
+    result_error_system_error,         /**< Some internal error happened. */
+    result_error_compression_error,    /**< Compression failed. */
+    result_error_decompression_error,  /**< Decompression failed. */
+    result_error_round_trip_error,     /**< Data failed to round trip. */
+} result_error_t;
+
+/**
+ * The success type.
+ */
+typedef struct {
+    size_t total_size;  /**< The total compressed size. */
+} result_data_t;
+
+/**
+ * The result type.
+ * Do not access the member variables directory, use the helper functions.
+ */
+typedef struct {
+    result_error_t internal_error;
+    result_data_t internal_data;
+} result_t;
+
+/**
+ * Create a result of the error type.
+ */
+static result_t result_error(result_error_t error);
+/**
+ * Create a result of the success type.
+ */
+static result_t result_data(result_data_t data);
+
+/**
+ * Check if the result is an error or skip.
+ */
+static int result_is_error(result_t result);
+/**
+ * Check if the result error is skip.
+ */
+static int result_is_skip(result_t result);
+/**
+ * Get the result error or okay.
+ */
+static result_error_t result_get_error(result_t result);
+/**
+ * Get the result data. The result MUST be checked with result_is_error() first.
+ */
+static result_data_t result_get_data(result_t result);
+
+static result_t result_error(result_error_t error) {
+    result_t result = {
+        .internal_error = error,
+    };
+    return result;
+}
+
+static result_t result_data(result_data_t data) {
+    result_t result = {
+        .internal_error = result_error_ok,
+        .internal_data = data,
+    };
+    return result;
+}
+
+static int result_is_error(result_t result) {
+    return result_get_error(result) != result_error_ok;
+}
+
+static int result_is_skip(result_t result) {
+    return result_get_error(result) == result_error_skip;
+}
+
+static result_error_t result_get_error(result_t result) {
+    return result.internal_error;
+}
+
+char const* result_get_error_string(result_t result);
+
+static result_data_t result_get_data(result_t result) {
+    return result.internal_data;
+}
+
+#endif

tests/regression/results.csv

@@ -0,0 +1,350 @@
+Data,                        Config,                      Method,                      Total compressed size
+silesia.tar,                 level -5,                    compress simple,             7160438
+silesia.tar,                 level -3,                    compress simple,             6789024
+silesia.tar,                 level -1,                    compress simple,             6195462
+silesia.tar,                 level 0,                     compress simple,             4875071
+silesia.tar,                 level 1,                     compress simple,             5339697
+silesia.tar,                 level 3,                     compress simple,             4875071
+silesia.tar,                 level 4,                     compress simple,             4813104
+silesia.tar,                 level 5,                     compress simple,             4726961
+silesia.tar,                 level 6,                     compress simple,             4654401
+silesia.tar,                 level 7,                     compress simple,             4591933
+silesia.tar,                 level 9,                     compress simple,             4554098
+silesia.tar,                 level 13,                    compress simple,             4503496
+silesia.tar,                 level 16,                    compress simple,             4387233
+silesia.tar,                 level 19,                    compress simple,             4283123
+silesia,                     level -5,                    compress cctx,               7152294
+silesia,                     level -3,                    compress cctx,               6789969
+silesia,                     level -1,                    compress cctx,               6191548
+silesia,                     level 0,                     compress cctx,               4862377
+silesia,                     level 1,                     compress cctx,               5318036
+silesia,                     level 3,                     compress cctx,               4862377
+silesia,                     level 4,                     compress cctx,               4800629
+silesia,                     level 5,                     compress cctx,               4715005
+silesia,                     level 6,                     compress cctx,               4644055
+silesia,                     level 7,                     compress cctx,               4581559
+silesia,                     level 9,                     compress cctx,               4543862
+silesia,                     level 13,                    compress cctx,               4493931
+silesia,                     level 16,                    compress cctx,               4381885
+silesia,                     level 19,                    compress cctx,               4296899
+github,                      level -5,                    compress cctx,               232744
+github,                      level -5 with dict,          compress cctx,               45704
+github,                      level -3,                    compress cctx,               220611
+github,                      level -3 with dict,          compress cctx,               44510
+github,                      level -1,                    compress cctx,               176575
+github,                      level -1 with dict,          compress cctx,               41586
+github,                      level 0,                     compress cctx,               136397
+github,                      level 0 with dict,           compress cctx,               38700
+github,                      level 1,                     compress cctx,               143457
+github,                      level 1 with dict,           compress cctx,               41538
+github,                      level 3,                     compress cctx,               136397
+github,                      level 3 with dict,           compress cctx,               38700
+github,                      level 4,                     compress cctx,               136144
+github,                      level 4 with dict,           compress cctx,               38639
+github,                      level 5,                     compress cctx,               135106
+github,                      level 5 with dict,           compress cctx,               38934
+github,                      level 6,                     compress cctx,               135108
+github,                      level 6 with dict,           compress cctx,               38628
+github,                      level 7,                     compress cctx,               135108
+github,                      level 7 with dict,           compress cctx,               38741
+github,                      level 9,                     compress cctx,               135108
+github,                      level 9 with dict,           compress cctx,               39335
+github,                      level 13,                    compress cctx,               133741
+github,                      level 13 with dict,          compress cctx,               39670
+github,                      level 16,                    compress cctx,               133741
+github,                      level 16 with dict,          compress cctx,               37928
+github,                      level 19,                    compress cctx,               133717
+github,                      level 19 with dict,          compress cctx,               37567
+silesia,                     level -5,                    zstdcli,                     7152342
+silesia,                     level -3,                    zstdcli,                     6790021
+silesia,                     level -1,                    zstdcli,                     6191597
+silesia,                     level 0,                     zstdcli,                     4862425
+silesia,                     level 1,                     zstdcli,                     5318084
+silesia,                     level 3,                     zstdcli,                     4862425
+silesia,                     level 4,                     zstdcli,                     4800677
+silesia,                     level 5,                     zstdcli,                     4715053
+silesia,                     level 6,                     zstdcli,                     4644103
+silesia,                     level 7,                     zstdcli,                     4581607
+silesia,                     level 9,                     zstdcli,                     4543910
+silesia,                     level 13,                    zstdcli,                     4493979
+silesia,                     level 16,                    zstdcli,                     4381933
+silesia,                     level 19,                    zstdcli,                     4296947
+silesia.tar,                 level -5,                    zstdcli,                     7159586
+silesia.tar,                 level -3,                    zstdcli,                     6791018
+silesia.tar,                 level -1,                    zstdcli,                     6196283
+silesia.tar,                 level 0,                     zstdcli,                     4876730
+silesia.tar,                 level 1,                     zstdcli,                     5340312
+silesia.tar,                 level 3,                     zstdcli,                     4876730
+silesia.tar,                 level 4,                     zstdcli,                     4817723
+silesia.tar,                 level 5,                     zstdcli,                     4730389
+silesia.tar,                 level 6,                     zstdcli,                     4655708
+silesia.tar,                 level 7,                     zstdcli,                     4593407
+silesia.tar,                 level 9,                     zstdcli,                     4556135
+silesia.tar,                 level 13,                    zstdcli,                     4503500
+silesia.tar,                 level 16,                    zstdcli,                     4387237
+silesia.tar,                 level 19,                    zstdcli,                     4283127
+silesia.tar,                 no source size,              zstdcli,                     4876726
+github,                      level -5,                    zstdcli,                     234744
+github,                      level -5 with dict,          zstdcli,                     47528
+github,                      level -3,                    zstdcli,                     222611
+github,                      level -3 with dict,          zstdcli,                     46394
+github,                      level -1,                    zstdcli,                     178575
+github,                      level -1 with dict,          zstdcli,                     43401
+github,                      level 0,                     zstdcli,                     138397
+github,                      level 0 with dict,           zstdcli,                     40316
+github,                      level 1,                     zstdcli,                     145457
+github,                      level 1 with dict,           zstdcli,                     43242
+github,                      level 3,                     zstdcli,                     138397
+github,                      level 3 with dict,           zstdcli,                     40316
+github,                      level 4,                     zstdcli,                     138144
+github,                      level 4 with dict,           zstdcli,                     40292
+github,                      level 5,                     zstdcli,                     137106
+github,                      level 5 with dict,           zstdcli,                     40938
+github,                      level 6,                     zstdcli,                     137108
+github,                      level 6 with dict,           zstdcli,                     40632
+github,                      level 7,                     zstdcli,                     137108
+github,                      level 7 with dict,           zstdcli,                     40766
+github,                      level 9,                     zstdcli,                     137108
+github,                      level 9 with dict,           zstdcli,                     41326
+github,                      level 13,                    zstdcli,                     135741
+github,                      level 13 with dict,          zstdcli,                     41670
+github,                      level 16,                    zstdcli,                     135741
+github,                      level 16 with dict,          zstdcli,                     39940
+github,                      level 19,                    zstdcli,                     135717
+github,                      level 19 with dict,          zstdcli,                     39576
+silesia,                     level -5,                    advanced one pass,           7152294
+silesia,                     level -3,                    advanced one pass,           6789969
+silesia,                     level -1,                    advanced one pass,           6191548
+silesia,                     level 0,                     advanced one pass,           4862377
+silesia,                     level 1,                     advanced one pass,           5318036
+silesia,                     level 3,                     advanced one pass,           4862377
+silesia,                     level 4,                     advanced one pass,           4800629
+silesia,                     level 5,                     advanced one pass,           4715005
+silesia,                     level 6,                     advanced one pass,           4644055
+silesia,                     level 7,                     advanced one pass,           4581559
+silesia,                     level 9,                     advanced one pass,           4543862
+silesia,                     level 13,                    advanced one pass,           4493931
+silesia,                     level 16,                    advanced one pass,           4381885
+silesia,                     level 19,                    advanced one pass,           4296899
+silesia,                     no source size,              advanced one pass,           4862377
+silesia.tar,                 level -5,                    advanced one pass,           7160438
+silesia.tar,                 level -3,                    advanced one pass,           6789024
+silesia.tar,                 level -1,                    advanced one pass,           6195462
+silesia.tar,                 level 0,                     advanced one pass,           4875071
+silesia.tar,                 level 1,                     advanced one pass,           5339697
+silesia.tar,                 level 3,                     advanced one pass,           4875071
+silesia.tar,                 level 4,                     advanced one pass,           4813104
+silesia.tar,                 level 5,                     advanced one pass,           4726961
+silesia.tar,                 level 6,                     advanced one pass,           4654401
+silesia.tar,                 level 7,                     advanced one pass,           4591933
+silesia.tar,                 level 9,                     advanced one pass,           4554098
+silesia.tar,                 level 13,                    advanced one pass,           4503496
+silesia.tar,                 level 16,                    advanced one pass,           4387233
+silesia.tar,                 level 19,                    advanced one pass,           4283123
+silesia.tar,                 no source size,              advanced one pass,           4875071
+github,                      level -5,                    advanced one pass,           232744
+github,                      level -5 with dict,          advanced one pass,           45528
+github,                      level -3,                    advanced one pass,           220611
+github,                      level -3 with dict,          advanced one pass,           44394
+github,                      level -1,                    advanced one pass,           176575
+github,                      level -1 with dict,          advanced one pass,           41401
+github,                      level 0,                     advanced one pass,           136397
+github,                      level 0 with dict,           advanced one pass,           38316
+github,                      level 1,                     advanced one pass,           143457
+github,                      level 1 with dict,           advanced one pass,           41242
+github,                      level 3,                     advanced one pass,           136397
+github,                      level 3 with dict,           advanced one pass,           38316
+github,                      level 4,                     advanced one pass,           136144
+github,                      level 4 with dict,           advanced one pass,           38292
+github,                      level 5,                     advanced one pass,           135106
+github,                      level 5 with dict,           advanced one pass,           38938
+github,                      level 6,                     advanced one pass,           135108
+github,                      level 6 with dict,           advanced one pass,           38632
+github,                      level 7,                     advanced one pass,           135108
+github,                      level 7 with dict,           advanced one pass,           38766
+github,                      level 9,                     advanced one pass,           135108
+github,                      level 9 with dict,           advanced one pass,           39326
+github,                      level 13,                    advanced one pass,           133741
+github,                      level 13 with dict,          advanced one pass,           39670
+github,                      level 16,                    advanced one pass,           133741
+github,                      level 16 with dict,          advanced one pass,           37940
+github,                      level 19,                    advanced one pass,           133717
+github,                      level 19 with dict,          advanced one pass,           37576
+github,                      no source size,              advanced one pass,           136397
+silesia,                     level -5,                    advanced one pass small out, 7152294
+silesia,                     level -3,                    advanced one pass small out, 6789969
+silesia,                     level -1,                    advanced one pass small out, 6191548
+silesia,                     level 0,                     advanced one pass small out, 4862377
+silesia,                     level 1,                     advanced one pass small out, 5318036
+silesia,                     level 3,                     advanced one pass small out, 4862377
+silesia,                     level 4,                     advanced one pass small out, 4800629
+silesia,                     level 5,                     advanced one pass small out, 4715005
+silesia,                     level 6,                     advanced one pass small out, 4644055
+silesia,                     level 7,                     advanced one pass small out, 4581559
+silesia,                     level 9,                     advanced one pass small out, 4543862
+silesia,                     level 13,                    advanced one pass small out, 4493931
+silesia,                     level 16,                    advanced one pass small out, 4381885
+silesia,                     level 19,                    advanced one pass small out, 4296899
+silesia,                     no source size,              advanced one pass small out, 4862377
+silesia.tar,                 level -5,                    advanced one pass small out, 7160438
+silesia.tar,                 level -3,                    advanced one pass small out, 6789024
+silesia.tar,                 level -1,                    advanced one pass small out, 6195462
+silesia.tar,                 level 0,                     advanced one pass small out, 4875071
+silesia.tar,                 level 1,                     advanced one pass small out, 5339697
+silesia.tar,                 level 3,                     advanced one pass small out, 4875071
+silesia.tar,                 level 4,                     advanced one pass small out, 4813104
+silesia.tar,                 level 5,                     advanced one pass small out, 4726961
+silesia.tar,                 level 6,                     advanced one pass small out, 4654401
+silesia.tar,                 level 7,                     advanced one pass small out, 4591933
+silesia.tar,                 level 9,                     advanced one pass small out, 4554098
+silesia.tar,                 level 13,                    advanced one pass small out, 4503496
+silesia.tar,                 level 16,                    advanced one pass small out, 4387233
+silesia.tar,                 level 19,                    advanced one pass small out, 4283123
+silesia.tar,                 no source size,              advanced one pass small out, 4875071
+github,                      level -5,                    advanced one pass small out, 232744
+github,                      level -5 with dict,          advanced one pass small out, 45528
+github,                      level -3,                    advanced one pass small out, 220611
+github,                      level -3 with dict,          advanced one pass small out, 44394
+github,                      level -1,                    advanced one pass small out, 176575
+github,                      level -1 with dict,          advanced one pass small out, 41401
+github,                      level 0,                     advanced one pass small out, 136397
+github,                      level 0 with dict,           advanced one pass small out, 38316
+github,                      level 1,                     advanced one pass small out, 143457
+github,                      level 1 with dict,           advanced one pass small out, 41242
+github,                      level 3,                     advanced one pass small out, 136397
+github,                      level 3 with dict,           advanced one pass small out, 38316
+github,                      level 4,                     advanced one pass small out, 136144
+github,                      level 4 with dict,           advanced one pass small out, 38292
+github,                      level 5,                     advanced one pass small out, 135106
+github,                      level 5 with dict,           advanced one pass small out, 38938
+github,                      level 6,                     advanced one pass small out, 135108
+github,                      level 6 with dict,           advanced one pass small out, 38632
+github,                      level 7,                     advanced one pass small out, 135108
+github,                      level 7 with dict,           advanced one pass small out, 38766
+github,                      level 9,                     advanced one pass small out, 135108
+github,                      level 9 with dict,           advanced one pass small out, 39326
+github,                      level 13,                    advanced one pass small out, 133741
+github,                      level 13 with dict,          advanced one pass small out, 39670
+github,                      level 16,                    advanced one pass small out, 133741
+github,                      level 16 with dict,          advanced one pass small out, 37940
+github,                      level 19,                    advanced one pass small out, 133717
+github,                      level 19 with dict,          advanced one pass small out, 37576
+github,                      no source size,              advanced one pass small out, 136397
+silesia,                     level -5,                    advanced streaming,          7152294
+silesia,                     level -3,                    advanced streaming,          6789973
+silesia,                     level -1,                    advanced streaming,          6191549
+silesia,                     level 0,                     advanced streaming,          4862377
+silesia,                     level 1,                     advanced streaming,          5318036
+silesia,                     level 3,                     advanced streaming,          4862377
+silesia,                     level 4,                     advanced streaming,          4800629
+silesia,                     level 5,                     advanced streaming,          4715005
+silesia,                     level 6,                     advanced streaming,          4644055
+silesia,                     level 7,                     advanced streaming,          4581559
+silesia,                     level 9,                     advanced streaming,          4543862
+silesia,                     level 13,                    advanced streaming,          4493931
+silesia,                     level 16,                    advanced streaming,          4381885
+silesia,                     level 19,                    advanced streaming,          4296899
+silesia,                     no source size,              advanced streaming,          4862341
+silesia.tar,                 level -5,                    advanced streaming,          7160440
+silesia.tar,                 level -3,                    advanced streaming,          6789026
+silesia.tar,                 level -1,                    advanced streaming,          6195465
+silesia.tar,                 level 0,                     advanced streaming,          4875071
+silesia.tar,                 level 1,                     advanced streaming,          5339701
+silesia.tar,                 level 3,                     advanced streaming,          4875071
+silesia.tar,                 level 4,                     advanced streaming,          4813104
+silesia.tar,                 level 5,                     advanced streaming,          4726977
+silesia.tar,                 level 6,                     advanced streaming,          4654404
+silesia.tar,                 level 7,                     advanced streaming,          4591934
+silesia.tar,                 level 9,                     advanced streaming,          4554105
+silesia.tar,                 level 13,                    advanced streaming,          4503496
+silesia.tar,                 level 16,                    advanced streaming,          4387233
+silesia.tar,                 level 19,                    advanced streaming,          4283123
+silesia.tar,                 no source size,              advanced streaming,          4875067
+github,                      level -5,                    advanced streaming,          232744
+github,                      level -5 with dict,          advanced streaming,          45528
+github,                      level -3,                    advanced streaming,          220611
+github,                      level -3 with dict,          advanced streaming,          44394
+github,                      level -1,                    advanced streaming,          176575
+github,                      level -1 with dict,          advanced streaming,          41401
+github,                      level 0,                     advanced streaming,          136397
+github,                      level 0 with dict,           advanced streaming,          38316
+github,                      level 1,                     advanced streaming,          143457
+github,                      level 1 with dict,           advanced streaming,          41242
+github,                      level 3,                     advanced streaming,          136397
+github,                      level 3 with dict,           advanced streaming,          38316
+github,                      level 4,                     advanced streaming,          136144
+github,                      level 4 with dict,           advanced streaming,          38292
+github,                      level 5,                     advanced streaming,          135106
+github,                      level 5 with dict,           advanced streaming,          38938
+github,                      level 6,                     advanced streaming,          135108
+github,                      level 6 with dict,           advanced streaming,          38632
+github,                      level 7,                     advanced streaming,          135108
+github,                      level 7 with dict,           advanced streaming,          38766
+github,                      level 9,                     advanced streaming,          135108
+github,                      level 9 with dict,           advanced streaming,          39326
+github,                      level 13,                    advanced streaming,          133741
+github,                      level 13 with dict,          advanced streaming,          39670
+github,                      level 16,                    advanced streaming,          133741
+github,                      level 16 with dict,          advanced streaming,          37940
+github,                      level 19,                    advanced streaming,          133717
+github,                      level 19 with dict,          advanced streaming,          37576
+github,                      no source size,              advanced streaming,          136397
+silesia,                     level -5,                    old streaming,               7152294
+silesia,                     level -3,                    old streaming,               6789973
+silesia,                     level -1,                    old streaming,               6191549
+silesia,                     level 0,                     old streaming,               4862377
+silesia,                     level 1,                     old streaming,               5318036
+silesia,                     level 3,                     old streaming,               4862377
+silesia,                     level 4,                     old streaming,               4800629
+silesia,                     level 5,                     old streaming,               4715005
+silesia,                     level 6,                     old streaming,               4644055
+silesia,                     level 7,                     old streaming,               4581559
+silesia,                     level 9,                     old streaming,               4543862
+silesia,                     level 13,                    old streaming,               4493931
+silesia,                     level 16,                    old streaming,               4381885
+silesia,                     level 19,                    old streaming,               4296899
+silesia,                     no source size,              old streaming,               4862341
+silesia.tar,                 level -5,                    old streaming,               7160440
+silesia.tar,                 level -3,                    old streaming,               6789026
+silesia.tar,                 level -1,                    old streaming,               6195465
+silesia.tar,                 level 0,                     old streaming,               4875071
+silesia.tar,                 level 1,                     old streaming,               5339701
+silesia.tar,                 level 3,                     old streaming,               4875071
+silesia.tar,                 level 4,                     old streaming,               4813104
+silesia.tar,                 level 5,                     old streaming,               4726977
+silesia.tar,                 level 6,                     old streaming,               4654404
+silesia.tar,                 level 7,                     old streaming,               4591934
+silesia.tar,                 level 9,                     old streaming,               4554105
+silesia.tar,                 level 13,                    old streaming,               4503496
+silesia.tar,                 level 16,                    old streaming,               4387233
+silesia.tar,                 level 19,                    old streaming,               4283123
+silesia.tar,                 no source size,              old streaming,               4875067
+github,                      level -5,                    old streaming,               232744
+github,                      level -5 with dict,          old streaming,               45528
+github,                      level -3,                    old streaming,               220611
+github,                      level -3 with dict,          old streaming,               44394
+github,                      level -1,                    old streaming,               176575
+github,                      level -1 with dict,          old streaming,               41401
+github,                      level 0,                     old streaming,               136397
+github,                      level 0 with dict,           old streaming,               38316
+github,                      level 1,                     old streaming,               143457
+github,                      level 1 with dict,           old streaming,               41242
+github,                      level 3,                     old streaming,               136397
+github,                      level 3 with dict,           old streaming,               38316
+github,                      level 4,                     old streaming,               136144
+github,                      level 4 with dict,           old streaming,               38292
+github,                      level 5,                     old streaming,               135106
+github,                      level 5 with dict,           old streaming,               38938
+github,                      level 6,                     old streaming,               135108
+github,                      level 6 with dict,           old streaming,               38632
+github,                      level 7,                     old streaming,               135108
+github,                      level 7 with dict,           old streaming,               38766
+github,                      level 9,                     old streaming,               135108
+github,                      level 9 with dict,           old streaming,               39326
+github,                      level 13,                    old streaming,               133741
+github,                      level 13 with dict,          old streaming,               39670
+github,                      level 16,                    old streaming,               133741
+github,                      level 16 with dict,          old streaming,               37940
+github,                      level 19,                    old streaming,               133717
+github,                      level 19 with dict,          old streaming,               37576
+github,                      no source size,              old streaming,               141003

tests/regression/test.c

@@ -0,0 +1,362 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <assert.h>
+#include <getopt.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "config.h"
+#include "data.h"
+#include "method.h"
+
+static int g_max_name_len = 0;
+
+/** Check if a name contains a comma or is too long. */
+static int is_name_bad(char const* name) {
+    if (name == NULL)
+        return 1;
+    int const len = strlen(name);
+    if (len > g_max_name_len)
+        g_max_name_len = len;
+    for (; *name != '\0'; ++name)
+        if (*name == ',')
+            return 1;
+    return 0;
+}
+
+/** Check if any of the names contain a comma. */
+static int are_names_bad() {
+    for (size_t method = 0; methods[method] != NULL; ++method)
+        if (is_name_bad(methods[method]->name)) {
+            fprintf(stderr, "method name %s is bad\n", methods[method]->name);
+            return 1;
+        }
+    for (size_t datum = 0; data[datum] != NULL; ++datum)
+        if (is_name_bad(data[datum]->name)) {
+            fprintf(stderr, "data name %s is bad\n", data[datum]->name);
+            return 1;
+        }
+    for (size_t config = 0; configs[config] != NULL; ++config)
+        if (is_name_bad(configs[config]->name)) {
+            fprintf(stderr, "config name %s is bad\n", configs[config]->name);
+            return 1;
+        }
+    return 0;
+}
+
+/**
+ * Option parsing using getopt.
+ * When you add a new option update: long_options, long_extras, and
+ * short_options.
+ */
+
+/** Option variables filled by parse_args. */
+static char const* g_output = NULL;
+static char const* g_diff = NULL;
+static char const* g_cache = NULL;
+static char const* g_zstdcli = NULL;
+static char const* g_config = NULL;
+static char const* g_data = NULL;
+static char const* g_method = NULL;
+
+typedef enum {
+    required_option,
+    optional_option,
+    help_option,
+} option_type;
+
+/**
+ * Extra state that we need to keep per-option that we can't store in getopt.
+ */
+struct option_extra {
+    int id; /**< The short option name, used as an id. */
+    char const* help; /**< The help message. */
+    option_type opt_type; /**< The option type: required, optional, or help. */
+    char const** value; /**< The value to set or NULL if no_argument. */
+};
+
+/** The options. */
+static struct option long_options[] = {
+    {"cache", required_argument, NULL, 'c'},
+    {"output", required_argument, NULL, 'o'},
+    {"zstd", required_argument, NULL, 'z'},
+    {"config", required_argument, NULL, 128},
+    {"data", required_argument, NULL, 129},
+    {"method", required_argument, NULL, 130},
+    {"diff", required_argument, NULL, 'd'},
+    {"help", no_argument, NULL, 'h'},
+};
+
+static size_t const nargs = sizeof(long_options) / sizeof(long_options[0]);
+
+/** The extra info for the options. Must be in the same order as the options. */
+static struct option_extra long_extras[] = {
+    {'c', "the cache directory", required_option, &g_cache},
+    {'o', "write the results here", required_option, &g_output},
+    {'z', "zstd cli tool", required_option, &g_zstdcli},
+    {128, "use this config", optional_option, &g_config},
+    {129, "use this data", optional_option, &g_data},
+    {130, "use this method", optional_option, &g_method},
+    {'d', "compare the results to this file", optional_option, &g_diff},
+    {'h', "display this message", help_option, NULL},
+};
+
+/** The short options. Must correspond to the options. */
+static char const short_options[] = "c:d:ho:z:";
+
+/** Return the help string for the option type. */
+static char const* required_message(option_type opt_type) {
+    switch (opt_type) {
+        case required_option:
+            return "[required]";
+        case optional_option:
+            return "[optional]";
+        case help_option:
+            return "";
+        default:
+            assert(0);
+            return NULL;
+    }
+}
+
+/** Print the help for the program. */
+static void print_help(void) {
+    fprintf(stderr, "regression test runner\n");
+    size_t const nargs = sizeof(long_options) / sizeof(long_options[0]);
+    for (size_t i = 0; i < nargs; ++i) {
+        if (long_options[i].val < 128) {
+            /* Long / short  - help [option type] */
+            fprintf(
+                stderr,
+                "--%s / -%c \t- %s %s\n",
+                long_options[i].name,
+                long_options[i].val,
+                long_extras[i].help,
+                required_message(long_extras[i].opt_type));
+        } else {
+            /* Short / long  - help [option type] */
+            fprintf(
+                stderr,
+                "--%s      \t- %s %s\n",
+                long_options[i].name,
+                long_extras[i].help,
+                required_message(long_extras[i].opt_type));
+        }
+    }
+}
+
+/** Parse the arguments. Teturn 0 on success. Print help on failure. */
+static int parse_args(int argc, char** argv) {
+    int option_index = 0;
+    int c;
+
+    while (1) {
+        c = getopt_long(argc, argv, short_options, long_options, &option_index);
+        if (c == -1)
+            break;
+
+        int found = 0;
+        for (size_t i = 0; i < nargs; ++i) {
+            if (c == long_extras[i].id && long_extras[i].value != NULL) {
+                *long_extras[i].value = optarg;
+                found = 1;
+                break;
+            }
+        }
+        if (found)
+            continue;
+
+        switch (c) {
+            case 'h':
+            case '?':
+            default:
+                print_help();
+                return 1;
+        }
+    }
+
+    int bad = 0;
+    for (size_t i = 0; i < nargs; ++i) {
+        if (long_extras[i].opt_type != required_option)
+            continue;
+        if (long_extras[i].value == NULL)
+            continue;
+        if (*long_extras[i].value != NULL)
+            continue;
+        fprintf(
+            stderr,
+            "--%s is a required argument but is not set\n",
+            long_options[i].name);
+        bad = 1;
+    }
+    if (bad) {
+        fprintf(stderr, "\n");
+        print_help();
+        return 1;
+    }
+
+    return 0;
+}
+
+/** Helper macro to print to stderr and a file. */
+#define tprintf(file, ...)            \
+    do {                              \
+        fprintf(file, __VA_ARGS__);   \
+        fprintf(stderr, __VA_ARGS__); \
+    } while (0)
+/** Helper macro to flush stderr and a file. */
+#define tflush(file)    \
+    do {                \
+        fflush(file);   \
+        fflush(stderr); \
+    } while (0)
+
+void tprint_names(
+    FILE* results,
+    char const* data_name,
+    char const* config_name,
+    char const* method_name) {
+    int const data_padding = g_max_name_len - strlen(data_name);
+    int const config_padding = g_max_name_len - strlen(config_name);
+    int const method_padding = g_max_name_len - strlen(method_name);
+
+    tprintf(
+        results,
+        "%s, %*s%s, %*s%s, %*s",
+        data_name,
+        data_padding,
+        "",
+        config_name,
+        config_padding,
+        "",
+        method_name,
+        method_padding,
+        "");
+}
+
+/**
+ * Run all the regression tests and record the results table to results and
+ * stderr progressively.
+ */
+static int run_all(FILE* results) {
+    tprint_names(results, "Data", "Config", "Method");
+    tprintf(results, "Total compressed size\n");
+    for (size_t method = 0; methods[method] != NULL; ++method) {
+        if (g_method != NULL && strcmp(methods[method]->name, g_method))
+            continue;
+        for (size_t datum = 0; data[datum] != NULL; ++datum) {
+            if (g_data != NULL && strcmp(data[datum]->name, g_data))
+                continue;
+            /* Create the state common to all configs */
+            method_state_t* state = methods[method]->create(data[datum]);
+            for (size_t config = 0; configs[config] != NULL; ++config) {
+                if (g_config != NULL && strcmp(configs[config]->name, g_config))
+                    continue;
+                if (config_skip_data(configs[config], data[datum]))
+                    continue;
+                /* Print the result for the (method, data, config) tuple. */
+                result_t const result =
+                    methods[method]->compress(state, configs[config]);
+                if (result_is_skip(result))
+                    continue;
+                tprint_names(
+                    results,
+                    data[datum]->name,
+                    configs[config]->name,
+                    methods[method]->name);
+                if (result_is_error(result)) {
+                    tprintf(results, "%s\n", result_get_error_string(result));
+                } else {
+                    tprintf(
+                        results,
+                        "%llu\n",
+                        (unsigned long long)result_get_data(result).total_size);
+                }
+                tflush(results);
+            }
+            methods[method]->destroy(state);
+        }
+    }
+    return 0;
+}
+
+/** memcmp() the old results file and the new results file. */
+static int diff_results(char const* actual_file, char const* expected_file) {
+    data_buffer_t const actual = data_buffer_read(actual_file);
+    data_buffer_t const expected = data_buffer_read(expected_file);
+    int ret = 1;
+
+    if (actual.data == NULL) {
+        fprintf(stderr, "failed to open results '%s' for diff\n", actual_file);
+        goto out;
+    }
+    if (expected.data == NULL) {
+        fprintf(
+            stderr,
+            "failed to open previous results '%s' for diff\n",
+            expected_file);
+        goto out;
+    }
+
+    ret = data_buffer_compare(actual, expected);
+    if (ret != 0) {
+        fprintf(
+            stderr,
+            "actual results '%s' does not match expected results '%s'\n",
+            actual_file,
+            expected_file);
+    } else {
+        fprintf(stderr, "actual results match expected results\n");
+    }
+out:
+    data_buffer_free(actual);
+    data_buffer_free(expected);
+    return ret;
+}
+
+int main(int argc, char** argv) {
+    /* Parse args and validate modules. */
+    int ret = parse_args(argc, argv);
+    if (ret != 0)
+        return ret;
+
+    if (are_names_bad())
+        return 1;
+
+    /* Initialize modules. */
+    method_set_zstdcli(g_zstdcli);
+    ret = data_init(g_cache);
+    if (ret != 0) {
+        fprintf(stderr, "data_init() failed with error=%s\n", strerror(ret));
+        return 1;
+    }
+
+    /* Run the regression tests. */
+    ret = 1;
+    FILE* results = fopen(g_output, "w");
+    if (results == NULL) {
+        fprintf(stderr, "Failed to open the output file\n");
+        goto out;
+    }
+    ret = run_all(results);
+    fclose(results);
+
+    if (ret != 0)
+        goto out;
+
+    if (g_diff)
+        /* Diff the new results with the previous results. */
+        ret = diff_results(g_output, g_diff);
+
+out:
+    data_finish();
+    return ret;
+}