[build] Add support for ASM files in Make + CMake

* Extract out common portion of `lib/Makefile` into `lib/libzstd.mk`.
  Most relevantly, the way we find library files.
* Use `lib/libzstd.mk` in the other Makefiles instead of repeating the
  same code.
* Add a test `tests/test-variants.sh` that checks that the builds of
  `make -C programs allVariants` are correct, and run it in Actions.
* Adds support for ASM files in the CMake build.

The Meson build is not updated because it lists every file in zstd,
and supports ASM off the bat, so the Huffman ASM commit will just add
the ASM file to the list.

The Visual Studios build is not updated because I'm not adding ASM
support to Visual Studios yet.

.circleci/config.yml

@@ -7,6 +7,8 @@ jobs:
     #       preinstalled to reduce installation time.
     docker:
       - image: fbopensource/zstd-circleci-primary:0.0.1
+    # TODO: Re-enable aarch64 build:
+    #       make aarch64build     && make clean
     steps:
       - checkout
       - run:
@@ -14,12 +16,11 @@ jobs:
           command: |
             ./tests/test-license.py
             cc -v; CFLAGS="-O0 -Werror -pedantic" make all && make clean
-            make c99build         ; make clean
+            make c99build         && make clean
-            make c11build         ; make clean
+            make c11build         && make clean
-            make aarch64build     ; make clean
+            make -j regressiontest&& make clean
-            make -j regressiontest; make clean
+            make shortest         && make clean
-            make shortest         ; make clean
+            make cxxtest          && make clean
-            make cxxtest          ; make clean
   # the second half of the jobs are in this test
   short-tests-1:
     docker:

.github/workflows/dev-short-tests.yml

@@ -200,6 +200,15 @@ jobs:
         make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
         make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
 
+  test-variants:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make all variants & validate
+      run: |
+        make -j -C programs allVariants MOREFLAGS=-O0
+        ./tests/test-variants.sh
+
 
   qemu-consistency:
     name: QEMU ${{ matrix.name }}
@@ -263,7 +272,6 @@ jobs:
       run: |
         LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
 
-
 # This test currently fails on Github Actions specifically.
 # Possible reason : TTY emulation.
 # Note that the same test works fine locally and on travisCI.

Makefile

@@ -217,7 +217,7 @@ armbuild: clean
 	CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd
 
 aarch64build: clean
-	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allzstd
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd
 
 ppcbuild: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
@@ -381,23 +381,23 @@ cmakebuild:
 
 c89build: clean
 	$(CC) -v
-	CFLAGS="-std=c89 -Werror" $(MAKE) allmost  # will fail, due to missing support for `long long`
+	CFLAGS="-std=c89 -Werror -O0" $(MAKE) allmost  # will fail, due to missing support for `long long`
 
 gnu90build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu90 -Werror" $(MAKE) allmost
+	CFLAGS="-std=gnu90 -Werror -O0" $(MAKE) allmost
 
 c99build: clean
 	$(CC) -v
-	CFLAGS="-std=c99 -Werror" $(MAKE) allmost
+	CFLAGS="-std=c99 -Werror -O0" $(MAKE) allmost
 
 gnu99build: clean
 	$(CC) -v
-	CFLAGS="-std=gnu99 -Werror" $(MAKE) allmost
+	CFLAGS="-std=gnu99 -Werror -O0" $(MAKE) allmost
 
 c11build: clean
 	$(CC) -v
-	CFLAGS="-std=c11 -Werror" $(MAKE) allmost
+	CFLAGS="-std=c11 -Werror -O0" $(MAKE) allmost
 
 bmix64build: clean
 	$(CC) -v

build/cmake/lib/CMakeLists.txt

@@ -7,7 +7,7 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-project(libzstd C)
+project(libzstd C ASM)
 
 set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
@@ -22,7 +22,7 @@ include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
 
 file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
 file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
-file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
+file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c ${LIBRARY_DIR}/decompress/*.S)
 file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
 
 set(Sources

contrib/freestanding_lib/freestanding.py

@@ -460,7 +460,8 @@ class Freestanding(object):
         print(*args, **kwargs)
 
     def _copy_file(self, lib_path):
-        if not (lib_path.endswith(".c") or lib_path.endswith(".h")):
+        suffixes = [".c", ".h", ".S"]
+        if not any((lib_path.endswith(suffix) for suffix in suffixes)):
             return
         if lib_path in SKIPPED_FILES:
             self._log(f"\tSkipping file: {lib_path}")

contrib/linux-kernel/test/Makefile

@@ -18,9 +18,13 @@ CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE
 LINUX_ZSTD_MODULE     := $(wildcard $(LINUX_ZSTDLIB)/*.c)
 LINUX_ZSTD_COMMON     := $(wildcard $(LINUX_ZSTDLIB)/common/*.c)
 LINUX_ZSTD_COMPRESS   := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c)
-LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c)
+LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c $(LINUX_ZSTDLIB)/decompress/*.S)
 LINUX_ZSTD_FILES      := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_COMPRESS) $(LINUX_ZSTD_DECOMPRESS)
-LINUX_ZSTD_OBJECTS    := $(LINUX_ZSTD_FILES:.c=.o)
+LINUX_ZSTD_OBJECTS0   := $(LINUX_ZSTD_FILES:.c=.o)
+LINUX_ZSTD_OBJECTS    := $(LINUX_ZSTD_OBJECTS0:.S=.o)
+
+%.o: %.S
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $^ -o $@
 
 liblinuxzstd.a: $(LINUX_ZSTD_OBJECTS)
 	$(AR) $(ARFLAGS) $@ $^

contrib/pzstd/Makefile

@@ -57,19 +57,6 @@ LD_COMMAND  = $(CXX) $^          $(ALL_LDFLAGS) $(LIBS) -pthread -o $@
 CC_COMMAND  = $(CC)  $(DEPFLAGS) $(ALL_CFLAGS)   -c $<  -o $@
 CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $<  -o $@
 
-# Get a list of all zstd files so we rebuild the static library when we need to
-ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \
-                    $(wildcard $(ZSTDDIR)/common/*.h)
-ZSTDCOMP_FILES   := $(wildcard $(ZSTDDIR)/compress/*.c) \
-                    $(wildcard $(ZSTDDIR)/compress/*.h)
-ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \
-                    $(wildcard $(ZSTDDIR)/decompress/*.h)
-ZSTDPROG_FILES   := $(wildcard $(PROGDIR)/*.c) \
-                    $(wildcard $(PROGDIR)/*.h)
-ZSTD_FILES       := $(wildcard $(ZSTDDIR)/*.h) \
-                    $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
-                    $(ZSTDPROG_FILES)
-
 # List all the pzstd source files so we can determine their dependencies
 PZSTD_SRCS  := $(wildcard *.cpp)
 PZSTD_TESTS := $(wildcard test/*.cpp)
@@ -189,7 +176,8 @@ roundtrip: test/RoundTripTest$(EXT)
 
 # Use the static library that zstd builds for simplicity and
 # so we get the compiler options correct
-$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
+.PHONY: $(ZSTDDIR)/libzstd.a
+$(ZSTDDIR)/libzstd.a:
 	CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a
 
 # Rules to build the tests

lib/Makefile

@@ -8,110 +8,13 @@
 # You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
-# Note: by default, the static library is built single-threaded and dynamic library is built
-# multi-threaded. It is possible to force multi or single threaded builds by appending
-# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded).
-.PHONY: default
-default: lib-release
-
-# define silent mode as default (verbose mode with V=1 or VERBOSE=1)
-$(V)$(VERBOSE).SILENT:
-
-# When cross-compiling from linux to windows,
-# one might need to specify TARGET_SYSTEM as "Windows."
-# Building from Fedora fails without it.
-# (but Ubuntu and Debian don't need to set anything)
-TARGET_SYSTEM ?= $(OS)
-
-# Version numbers
-LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
-LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
-LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
-LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
-LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
-LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
-LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
-LIBVER := $(shell echo $(LIBVER_SCRIPT))
-VERSION?= $(LIBVER)
-CCVER := $(shell $(CC) --version)
-
-# ZSTD_LIB_MINIFY is a helper variable that
-# configures a bunch of other variables to space-optimized defaults.
-ZSTD_LIB_MINIFY ?= 0
-ifneq ($(ZSTD_LIB_MINIFY), 0)
-  HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
-  ZSTD_LEGACY_SUPPORT ?= 0
-  ZSTD_LIB_DEPRECATED ?= 0
-  HUF_FORCE_DECOMPRESS_X1 ?= 1
-  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1
-  ZSTD_NO_INLINE ?= 1
-  ZSTD_STRIP_ERROR_STRINGS ?= 1
-ifneq ($(HAVE_CC_OZ), 0)
-    # Some compilers (clang) support an even more space-optimized setting.
-    CFLAGS += -Oz
-else
-    CFLAGS += -Os
-endif
-  CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
-            -DDYNAMIC_BMI2=0 -DNDEBUG
-else
-  CFLAGS += -O3
-endif
-
-DEBUGLEVEL ?= 0
-CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL)
-ifeq ($(TARGET_SYSTEM),Windows_NT)   # MinGW assumed
-  CPPFLAGS += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
-endif
-DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wpointer-arith \
-            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls -Wmissing-prototypes -Wc++-compat
-CFLAGS   += $(DEBUGFLAGS) $(MOREFLAGS)
-FLAGS     = $(CPPFLAGS) $(CFLAGS)
-
-CPPFLAGS_DYNLIB  = -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded
-LDFLAGS_DYNLIB   = -pthread
-CPPFLAGS_STATLIB =                    # static library build defaults to single-threaded
-
-HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
-GREP_OPTIONS ?=
-ifeq ($HAVE_COLORNEVER, 1)
-  GREP_OPTIONS += --color=never
-endif
-GREP = grep $(GREP_OPTIONS)
-SED_ERE_OPT ?= -E
-
-ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
-ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
-ZSTDDECOMP_FILES := $(sort $(wildcard decompress/*.c))
-ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c))
-ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
-ZSTD_FILES := $(ZSTDCOMMON_FILES)
-
-ifeq ($(findstring GCC,$(CCVER)),GCC)
-decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
-endif
-
 # Modules
 ZSTD_LIB_COMPRESSION ?= 1
 ZSTD_LIB_DECOMPRESSION ?= 1
 ZSTD_LIB_DICTBUILDER ?= 1
 ZSTD_LIB_DEPRECATED ?= 0
 
-# Legacy support
+# Input variables for libzstd.mk
-ZSTD_LEGACY_SUPPORT ?= 5
-ZSTD_LEGACY_MULTITHREADED_API ?= 0
-
-# Build size optimizations
-HUF_FORCE_DECOMPRESS_X1 ?= 0
-HUF_FORCE_DECOMPRESS_X2 ?= 0
-ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0
-ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
-ZSTD_NO_INLINE ?= 0
-ZSTD_STRIP_ERROR_STRINGS ?= 0
-
 ifeq ($(ZSTD_LIB_COMPRESSION), 0)
   ZSTD_LIB_DICTBUILDER = 0
   ZSTD_LIB_DEPRECATED = 0
@@ -122,86 +25,46 @@ ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
   ZSTD_LIB_DEPRECATED = 0
 endif
 
+include libzstd.mk
+
+ZSTD_FILES := $(ZSTD_COMMON_FILES) $(ZSTD_LEGACY_FILES)
+
 ifneq ($(ZSTD_LIB_COMPRESSION), 0)
-  ZSTD_FILES += $(ZSTDCOMP_FILES)
+  ZSTD_FILES += $(ZSTD_COMPRESS_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
-  ZSTD_FILES += $(ZSTDDECOMP_FILES)
+  ZSTD_FILES += $(ZSTD_DECOMPRESS_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DEPRECATED), 0)
-  ZSTD_FILES += $(ZDEPR_FILES)
+  ZSTD_FILES += $(ZSTD_DEPRECATED_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
-  ZSTD_FILES += $(ZDICT_FILES)
+  ZSTD_FILES += $(ZSTD_DICTBUILDER_FILES)
 endif
 
-ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
-  CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
-endif
-
-ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
-  CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
-endif
-
-ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT), 0)
-  CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
-endif
-
-ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG), 0)
-  CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
-endif
-
-ifneq ($(ZSTD_NO_INLINE), 0)
-  CFLAGS += -DZSTD_NO_INLINE
-endif
-
-ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
-  CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
-endif
-
-ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
-  CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
-endif
-
-ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
-ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
-  ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
-endif
-endif
-CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-
 ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES))
-ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o)
+ZSTD_LOCAL_OBJ0 := $(ZSTD_LOCAL_SRC:.c=.o)
+ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_OBJ0:.S=.o)
 
-ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated
+VERSION := $(ZSTD_VERSION)
-vpath %.c $(ZSTD_SUBDIR)
 
-UNAME := $(shell uname)
+# Note: by default, the static library is built single-threaded and dynamic library is built
+# multi-threaded. It is possible to force multi or single threaded builds by appending
+# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded).
+.PHONY: default
+default: lib-release
 
-ifndef BUILD_DIR
+CPPFLAGS_DYNLIB  = -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded
-ifeq ($(UNAME), Darwin)
+LDFLAGS_DYNLIB   = -pthread
-  ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0)
+CPPFLAGS_STATLIB =                    # static library build defaults to single-threaded
-    HASH ?= md5
+
-  endif
+
-else ifeq ($(UNAME), FreeBSD)
+ifeq ($(findstring GCC,$(CCVER)),GCC)
-  HASH ?= gmd5sum
+decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
-else ifeq ($(UNAME), NetBSD)
-  HASH ?= md5 -n
-else ifeq ($(UNAME), OpenBSD)
-  HASH ?= md5
 endif
-HASH ?= md5sum
-
-HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " " )
-HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
-ifeq ($(HAVE_HASH),0)
-  $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags)
-  BUILD_DIR := obj/generic_noconf
-endif
-endif # BUILD_DIR
 
 
 # macOS linker doesn't support -soname, and use different extension
@@ -218,13 +81,6 @@ else
   SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
 endif
 
-SET_CACHE_DIRECTORY = \
-   +$(MAKE) --no-print-directory $@ \
-    BUILD_DIR=obj/$(HASH_DIR) \
-    CPPFLAGS="$(CPPFLAGS)" \
-    CFLAGS="$(CFLAGS)" \
-    LDFLAGS="$(LDFLAGS)"
-
 
 .PHONY: all
 all: lib
@@ -233,6 +89,13 @@ all: lib
 .PHONY: libzstd.a  # must be run every time
 libzstd.a: CPPFLAGS += $(CPPFLAGS_STATLIB)
 
+SET_CACHE_DIRECTORY = \
+   +$(MAKE) --no-print-directory $@ \
+    BUILD_DIR=obj/$(HASH_DIR) \
+    CPPFLAGS="$(CPPFLAGS)" \
+    CFLAGS="$(CFLAGS)" \
+    LDFLAGS="$(LDFLAGS)"
+
 ifndef BUILD_DIR
 # determine BUILD_DIR from compilation flags
 
@@ -343,6 +206,14 @@ $(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR)
 	@echo CC $@
 	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
 
+$(ZSTD_DYNLIB_DIR)/%.o : %.S | $(ZSTD_DYNLIB_DIR)
+	@echo AS $@
+	$(COMPILE.c) $(OUTPUT_OPTION) $<
+
+$(ZSTD_STATLIB_DIR)/%.o : %.S | $(ZSTD_STATLIB_DIR)
+	@echo AS $@
+	$(COMPILE.c) $(OUTPUT_OPTION) $<
+
 MKDIR ?= mkdir
 $(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR):
 	$(MKDIR) -p $@

lib/compress/huf_compress.c

@@ -445,7 +445,7 @@ typedef struct {
 /* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing.
  * Strategy is to use as many buckets as possible for representing distinct
  * counts while using the remainder to represent all "large" counts.
- * 
+ *
  * To satisfy this requirement for 192 buckets, we can do the following:
  * Let buckets 0-166 represent distinct counts of [0, 166]
  * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing.
@@ -517,7 +517,7 @@ static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high)
 }
 
 /* Classic quicksort by descending with partially iterative calls
- * to reduce worst case callstack size. 
+ * to reduce worst case callstack size.
  */
 static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) {
     int const kInsertionSortThreshold = 8;
@@ -813,6 +813,7 @@ FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int id
         assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0);
         /* We didn't overwrite any bits in the bit container. */
         assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER);
+        (void)dirtyBits;
     }
 #endif
 }

lib/libzstd.mk

@@ -0,0 +1,185 @@
+# ################################################################
+# Copyright (c) Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+##################################################################
+# Input Variables
+##################################################################
+
+# Zstd lib directory
+LIBZSTD ?= ./
+
+# Legacy support
+ZSTD_LEGACY_SUPPORT ?= 5
+ZSTD_LEGACY_MULTITHREADED_API ?= 0
+
+# Build size optimizations
+HUF_FORCE_DECOMPRESS_X1 ?= 0
+HUF_FORCE_DECOMPRESS_X2 ?= 0
+ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 0
+ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG ?= 0
+ZSTD_NO_INLINE ?= 0
+ZSTD_STRIP_ERROR_STRINGS ?= 0
+
+# Assembly support
+ZSTD_NO_ASM ?= 0
+
+##################################################################
+# libzstd helpers
+##################################################################
+
+# Make 4.3 doesn't support '\#' anymore (https://lwn.net/Articles/810071/)
+NUM_SYMBOL := \#
+
+# define silent mode as default (verbose mode with V=1 or VERBOSE=1)
+$(V)$(VERBOSE).SILENT:
+
+# When cross-compiling from linux to windows,
+# one might need to specify TARGET_SYSTEM as "Windows."
+# Building from Fedora fails without it.
+# (but Ubuntu and Debian don't need to set anything)
+TARGET_SYSTEM ?= $(OS)
+
+# Version numbers
+LIBVER_SRC := $(LIBZSTD)/zstd.h
+LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
+LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
+LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
+LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
+LIBVER := $(shell echo $(LIBVER_SCRIPT))
+CCVER := $(shell $(CC) --version)
+ZSTD_VERSION?= $(LIBVER)
+
+# ZSTD_LIB_MINIFY is a helper variable that
+# configures a bunch of other variables to space-optimized defaults.
+ZSTD_LIB_MINIFY ?= 0
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+  HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
+  ZSTD_LEGACY_SUPPORT ?= 0
+  ZSTD_LIB_DEPRECATED ?= 0
+  HUF_FORCE_DECOMPRESS_X1 ?= 1
+  ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT ?= 1
+  ZSTD_NO_INLINE ?= 1
+  ZSTD_STRIP_ERROR_STRINGS ?= 1
+ifneq ($(HAVE_CC_OZ), 0)
+    # Some compilers (clang) support an even more space-optimized setting.
+    CFLAGS += -Oz
+else
+    CFLAGS += -Os
+endif
+  CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
+            -DDYNAMIC_BMI2=0 -DNDEBUG
+else
+  CFLAGS += -O3
+endif
+
+DEBUGLEVEL ?= 0
+CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL)
+ifeq ($(TARGET_SYSTEM),Windows_NT)   # MinGW assumed
+  CPPFLAGS += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
+endif
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls -Wmissing-prototypes -Wc++-compat
+CFLAGS   += $(DEBUGFLAGS) $(MOREFLAGS)
+LDFLAGS  += $(MOREFLAGS)
+FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+
+HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+GREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+  GREP_OPTIONS += --color=never
+endif
+GREP = grep $(GREP_OPTIONS)
+SED_ERE_OPT ?= -E
+
+ZSTD_COMMON_FILES := $(sort $(wildcard $(LIBZSTD)/common/*.c))
+ZSTD_COMPRESS_FILES := $(sort $(wildcard $(LIBZSTD)/compress/*.c))
+ZSTD_DECOMPRESS_FILES := $(sort $(wildcard $(LIBZSTD)/decompress/*.c))
+ZSTD_DICTBUILDER_FILES := $(sort $(wildcard $(LIBZSTD)/dictBuilder/*.c))
+ZSTD_DEPRECATED_FILES := $(sort $(wildcard $(LIBZSTD)/deprecated/*.c))
+ZSTD_LEGACY_FILES :=
+
+ZSTD_DECOMPRESS_AMD64_ASM_FILES := $(sort $(wildcard $(LIBZSTD)/decompress/*_amd64.S))
+
+ifneq ($(ZSTD_NO_ASM), 0)
+  CPPFLAGS += -DHUF_DISABLE_ASM
+else
+  # Unconditionally add the ASM files they are disabled by
+  # macros in the .S file.
+  ZSTD_DECOMPRESS_FILES += $(ZSTD_DECOMPRESS_AMD64_ASM_FILES)
+endif
+
+ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
+  CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
+endif
+
+ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
+  CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
+endif
+
+ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT), 0)
+  CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
+endif
+
+ifneq ($(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG), 0)
+  CFLAGS += -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
+endif
+
+ifneq ($(ZSTD_NO_INLINE), 0)
+  CFLAGS += -DZSTD_NO_INLINE
+endif
+
+ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
+  CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
+endif
+
+ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
+  CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
+endif
+
+ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
+ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
+  ZSTD_LEGACY_FILES += $(shell ls $(LIBZSTD)/legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+endif
+endif
+CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+
+UNAME := $(shell uname)
+
+ifndef BUILD_DIR
+ifeq ($(UNAME), Darwin)
+  ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0)
+    HASH ?= md5
+  endif
+else ifeq ($(UNAME), FreeBSD)
+  HASH ?= gmd5sum
+else ifeq ($(UNAME), NetBSD)
+  HASH ?= md5 -n
+else ifeq ($(UNAME), OpenBSD)
+  HASH ?= md5
+endif
+HASH ?= md5sum
+
+HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " " )
+HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
+ifeq ($(HAVE_HASH),0)
+  $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags)
+  BUILD_DIR := obj/generic_noconf
+endif
+endif # BUILD_DIR
+
+ZSTD_SUBDIR := $(LIBZSTD)/common $(LIBZSTD)/compress $(LIBZSTD)/decompress $(LIBZSTD)/dictBuilder $(LIBZSTD)/legacy $(LIBZSTD)/deprecated
+vpath %.c $(ZSTD_SUBDIR)
+vpath %.S $(ZSTD_SUBDIR)

programs/Makefile

@@ -9,7 +9,7 @@
 # ##########################################################################
 # zstd : Command Line Utility, supporting gzip-like arguments
 # zstd32 : Same as zstd, but forced to compile in 32-bits mode
-# zstd_nolegacy : zstd without support of decompression of legacy versions
+# zstd-nolegacy : zstd without support of decompression of legacy versions
 # zstd-small : minimal zstd without dictionary builder and benchmark
 # zstd-compress : compressor-only version of zstd
 # zstd-decompress : decompressor-only version of zstd
@@ -18,31 +18,9 @@
 .PHONY: default
 default: zstd-release
 
-# silent mode by default; verbose can be triggered by V=1 or VERBOSE=1
+LIBZSTD := ../lib
-$(V)$(VERBOSE).SILENT:
 
-
+include $(LIBZSTD)/libzstd.mk
-ZSTDDIR := ../lib
-
-# Version numbers
-LIBVER_SRC := $(ZSTDDIR)/zstd.h
-LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
-LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
-LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
-LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
-LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
-LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
-LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
-LIBVER  := $(shell echo $(LIBVER_SCRIPT))
-
-ZSTD_VERSION = $(LIBVER)
-
-HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
-GREP_OPTIONS ?=
-ifeq ($HAVE_COLORNEVER, 1)
-  GREP_OPTIONS += --color=never
-endif
-GREP = grep $(GREP_OPTIONS)
 
 ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
   ALIGN_LOOP = -falign-loops=32
@@ -50,78 +28,25 @@ else
   ALIGN_LOOP =
 endif
 
-DEBUGLEVEL ?= 0
+ZSTDLIB_COMMON_SRC := $(sort $(ZSTD_COMMON_FILES))
-CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL)
+ZSTDLIB_COMPRESS_SRC := $(sort $(ZSTD_COMPRESS_FILES))
-ifeq ($(OS),Windows_NT)   # MinGW assumed
+ZSTDLIB_DECOMPRESS_SRC := $(sort $(ZSTD_DECOMPRESS_FILES))
-  CPPFLAGS += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
+ZSTDLIB_CORE_SRC := $(sort $(ZSTD_DECOMPRESS_FILES) $(ZSTD_COMMON_FILES) $(ZSTD_COMPRESS_FILES))
-endif
+ZDICT_SRC := $(sort $(ZSTD_DICTBUILDER_FILES))
-CFLAGS   ?= -O3
+ZSTDLEGACY_SRC := $(sort $(ZSTD_LEGACY_FILES))
-DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wpointer-arith \
-            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls -Wmissing-prototypes -Wc++-compat
-CFLAGS   += $(DEBUGFLAGS)
-CPPFLAGS += $(MOREFLAGS)
-LDFLAGS  += $(MOREFLAGS)
-FLAGS     = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
-
-ZSTDLIB_COMMON := $(ZSTDDIR)/common
-ZSTDLIB_COMPRESS := $(ZSTDDIR)/compress
-ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress
-ZDICT_DIR := $(ZSTDDIR)/dictBuilder
-ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy
-
-vpath %.c $(ZSTDLIB_COMMON) $(ZSTDLIB_COMPRESS) $(ZSTDLIB_DECOMPRESS) $(ZDICT_DIR) $(ZSTDLEGACY_DIR)
-
-ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c)
-ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c)
-ZSTDLIB_DECOMPRESS_C := $(wildcard $(ZSTDLIB_DECOMPRESS)/*.c)
-ZSTDLIB_CORE_SRC := $(ZSTDLIB_DECOMPRESS_C) $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C)
-ZDICT_SRC := $(wildcard $(ZDICT_DIR)/*.c)
-
-ZSTD_LEGACY_SUPPORT ?= 5
-ZSTDLEGACY_SRC :=
-ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
-ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
-  ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
-endif
-endif
 
 # Sort files in alphabetical order for reproducible builds
 ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC))
 ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC))
-ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_SRC:.c=.o)
+ZSTDLIB_LOCAL_OBJ0 := $(ZSTDLIB_LOCAL_SRC:.c=.o)
+ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_OBJ0:.S=.o)
 
 ZSTD_CLI_SRC := $(wildcard *.c)
 ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o)
 
 ZSTD_ALL_SRC = $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
-ZSTD_ALL_OBJ := $(ZSTD_ALL_SRC:.c=.o)
+ZSTD_ALL_OBJ0 := $(ZSTD_ALL_SRC:.c=.o)
-
+ZSTD_ALL_OBJ := $(ZSTD_ALL_OBJ0:.S=.o)
-UNAME := $(shell uname)
-
-ifndef BUILD_DIR
-ifeq ($(UNAME), Darwin)
-  ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0)
-    HASH ?= md5
-  endif
-else ifeq ($(UNAME), FreeBSD)
-  HASH ?= gmd5sum
-else ifeq ($(UNAME), NetBSD)
-  HASH ?= md5 -n
-else ifeq ($(UNAME), OpenBSD)
-  HASH ?= md5
-endif
-HASH ?= md5sum
-HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
-
-HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(LDLIBS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " ")
-ifeq ($(HAVE_HASH),0)
-  $(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags)
-  BUILD_DIR := obj/generic_noconf
-endif
-endif # BUILD_DIR
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
@@ -139,9 +64,6 @@ endif
 
 VOID = /dev/null
 
-# Make 4.3 doesn't support '\#' anymore (https://lwn.net/Articles/810071/)
-NUM_SYMBOL := \#
-
 # thread detection
 NO_THREAD_MSG := ==> no threads, building without multithreading support
 HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
@@ -212,7 +134,7 @@ SET_CACHE_DIRECTORY = \
 all: zstd
 
 .PHONY: allVariants
-allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder
+allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-frugal zstd-nolegacy zstd-dictBuilder
 
 .PHONY: zstd  # must always be run
 zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
@@ -276,6 +198,7 @@ zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC)
 
 ## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed
 zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
+zstd-nolegacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0
 zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ)
 	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
 
@@ -299,7 +222,7 @@ zstd-noxz : zstd
 
 ## zstd-dll: zstd executable linked to dynamic library libzstd (must have same version)
 .PHONY: zstd-dll
-zstd-dll : LDFLAGS+= -L$(ZSTDDIR)
+zstd-dll : LDFLAGS+= -L$(LIBZSTD)
 zstd-dll : LDLIBS += -lzstd
 zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c
 zstd-dll : zstd
@@ -323,18 +246,17 @@ zstd-pgo :
 ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
 zstd-small: CFLAGS = -Os -s
 zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c
-	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE $^ -o $@$(EXT)
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
 
-zstd-decompress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_DECOMPRESS_C) zstdcli.c util.c timefn.c fileio.c
+zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c
-	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
 
-zstd-compress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) zstdcli.c util.c timefn.c fileio.c
+zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c
-	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
 
 ## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
-zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE
+zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c
-zstd-dictBuilder: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c
+	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
-	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
 zstdmt: zstd
 	ln -sf zstd zstdmt
@@ -398,6 +320,10 @@ $(BUILD_DIR)/%.o : %.c $(BUILD_DIR)/%.d | $(BUILD_DIR)
 	@echo CC $@
 	$(COMPILE.c) $(DEPFLAGS) $(BUILD_DIR)/$*.d $(OUTPUT_OPTION) $<
 
+$(BUILD_DIR)/%.o : %.S | $(BUILD_DIR)
+	@echo AS $@
+	$(COMPILE.c) $(OUTPUT_OPTION) $<
+
 MKDIR ?= mkdir
 $(BUILD_DIR): ; $(MKDIR) -p $@
 

tests/Makefile

@@ -1,4 +1,5 @@
-# ################################################################
+
+ # ################################################################
 # Copyright (c) Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
@@ -19,46 +20,43 @@
 # zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode
 # ##########################################################################
 
-ZSTDDIR = ../lib
+LIBZSTD = ../lib
+
+ZSTD_LEGACY_SUPPORT ?= 0
+
+DEBUGLEVEL ?= 2
+export DEBUGLEVEL  # transmit value to sub-makefiles
+
+include $(LIBZSTD)/libzstd.mk
+
+ZSTDDIR = $(LIBZSTD)
 PRGDIR  = ../programs
 PYTHON ?= python3
 TESTARTEFACT := versionsTest
 
-DEBUGLEVEL ?= 2
+DEBUGFLAGS += -g -Wno-c++-compat
-export DEBUGLEVEL  # transmit value to sub-makefiles
-DEBUGFLAGS  = -g -DDEBUGLEVEL=$(DEBUGLEVEL)
 CPPFLAGS   += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
               -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
 			  -DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1
-ifeq ($(OS),Windows_NT)   # MinGW assumed
-CPPFLAGS   += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
-endif
-CFLAGS     ?= -O3
-CFLAGS     += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
-              -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-              -Wstrict-prototypes -Wundef                                     \
-              -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
-              -Wredundant-decls -Wmissing-prototypes -Wno-deprecated-declarations
-CFLAGS     += $(DEBUGFLAGS)
-CPPFLAGS   += $(MOREFLAGS)
 
-
+ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES))
-ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
+ZSTDCOMP_FILES   := $(sort $(ZSTD_COMPRESS_FILES))
-ZSTDCOMP_FILES   := $(ZSTDDIR)/compress/*.c
+ZSTDDECOMP_FILES := $(sort $(ZSTD_DECOMPRESS_FILES))
-ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
 ZSTD_FILES  := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
-ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
+ZDICT_FILES := $(sort $(ZSTD_DICTBUILDER_FILES))
 
 ZSTD_F1 := $(wildcard $(ZSTD_FILES))
 ZSTD_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdm_,$(ZSTD_F1))
 ZSTD_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdc_,$(ZSTD_OBJ1))
 ZSTD_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdd_,$(ZSTD_OBJ2))
-ZSTD_OBJECTS := $(ZSTD_OBJ3:.c=.o)
+ZSTD_OBJ4 := $(ZSTD_OBJ3:.c=.o)
+ZSTD_OBJECTS := $(ZSTD_OBJ4:.S=.o)
 
 ZSTDMT_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdmt_m_,$(ZSTD_F1))
 ZSTDMT_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdmt_c_,$(ZSTDMT_OBJ1))
 ZSTDMT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdmt_d_,$(ZSTDMT_OBJ2))
-ZSTDMT_OBJECTS := $(ZSTDMT_OBJ3:.c=.o)
+ZSTDMT_OBJ4 := $(ZSTDMT_OBJ3:.c=.o)
+ZSTDMT_OBJECTS := $(ZSTDMT_OBJ4:.S=.o)
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
@@ -119,6 +117,9 @@ zstdc_%.o : $(ZSTDDIR)/compress/%.c
 zstdd_%.o : $(ZSTDDIR)/decompress/%.c
 	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
 
+zstdd_%.o : $(ZSTDDIR)/decompress/%.S
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+
 zstdmt%.o : CPPFLAGS += $(MULTITHREAD_CPP)
 
 zstdmt_m_%.o : $(ZSTDDIR)/common/%.c
@@ -130,6 +131,9 @@ zstdmt_c_%.o : $(ZSTDDIR)/compress/%.c
 zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
 	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
 
+zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.S
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+
 fullbench32: CPPFLAGS += -m32
 fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
 fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
@@ -201,7 +205,7 @@ bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c
 
 invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
 
-legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4
+legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4
 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
 
 decodecorpus : LDLIBS += -lm

tests/fuzz/Makefile

@@ -23,6 +23,12 @@ else
 endif
 CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/
 
+LIBZSTD = ../../lib
+DEBUGLEVEL ?= 2
+ZSTD_LEGACY_SUPPORT ?= 1
+
+include $(LIBZSTD)/libzstd.mk
+
 ZSTDDIR = ../../lib
 PRGDIR = ../../programs
 CONTRIBDIR = ../../contrib
@@ -34,7 +40,7 @@ FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
 	-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
 	-Wstrict-prototypes -Wundef \
 	-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-	-Wredundant-decls \
+	-Wredundant-decls -Wno-deprecated-declarations \
 	-g -fno-omit-frame-pointer
 FUZZ_CFLAGS := $(FUZZ_EXTRA_FLAGS) $(CFLAGS)
 FUZZ_CXXFLAGS := $(FUZZ_EXTRA_FLAGS) -std=c++11 $(CXXFLAGS)
@@ -50,11 +56,11 @@ FUZZ_SRC := $(PRGDIR)/util.c ./fuzz_helpers.c ./zstd_helpers.c ./fuzz_data_produ
 SEEKABLE_HEADERS = $(CONTRIBDIR)/seekable_format/zstd_seekable.h
 SEEKABLE_OBJS = $(CONTRIBDIR)/seekable_format/zstdseek_compress.c $(CONTRIBDIR)/seekable_format/zstdseek_decompress.c
 
-ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c
+ZSTDCOMMON_SRC := $(ZSTD_COMMON_FILES)
-ZSTDCOMP_SRC   := $(ZSTDDIR)/compress/*.c
+ZSTDCOMP_SRC   := $(ZSTD_COMPRESS_FILES)
-ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c
+ZSTDDECOMP_SRC := $(ZSTD_DECOMPRESS_FILES)
-ZSTDDICT_SRC := $(ZSTDDIR)/dictBuilder/*.c
+ZSTDDICT_SRC := $(ZSTD_DICTBUILDER_FILES)
-ZSTDLEGACY_SRC := $(ZSTDDIR)/legacy/*.c
+ZSTDLEGACY_SRC := $(ZSTD_LEGACY_FILES)
 FUZZ_SRC       := \
 	$(FUZZ_SRC) \
 	$(ZSTDDECOMP_SRC) \
@@ -71,7 +77,8 @@ FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3)
 FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4))
 FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5))
 FUZZ_D_OBJ7 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ6))
-FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ7:.c=.o)
+FUZZ_D_OBJ8 := $(FUZZ_D_OBJ7:.c=.o)
+FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ8:.S=.o)
 
 FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC))
 FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1))
@@ -80,7 +87,8 @@ FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OB
 FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4))
 FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5))
 FUZZ_RT_OBJ7 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ6))
-FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ7:.c=.o)
+FUZZ_RT_OBJ8 := $(FUZZ_RT_OBJ7:.c=.o)
+FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ8:.S=.o)
 
 .PHONY: default all clean cleanall
 
@@ -117,6 +125,9 @@ rt_lib_compress_%.o: $(ZSTDDIR)/compress/%.c
 rt_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
 
+rt_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.S
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
+
 rt_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
 
@@ -138,6 +149,9 @@ d_lib_compress_%.o: $(ZSTDDIR)/compress/%.c
 d_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
 
+d_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.S
+	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
+
 d_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c
 	$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
 

tests/fuzzer.c

@@ -1528,7 +1528,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
             DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
             staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
-            DISPLAYLEVEL(4, "staticCCtxBuffer = %p,  staticCCtx = %p , ", staticCCtxBuffer, staticCCtx);
+            DISPLAYLEVEL(4, "staticCCtxBuffer = %p,  staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx);
             if (staticCCtx == NULL) goto _output_error;
             DISPLAYLEVEL(3, "OK \n");
 

tests/regression/Makefile

@@ -46,6 +46,7 @@ result.o: result.c result.h
 test.o: test.c data.h config.h method.h
 	$(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@
 
+.PHONY: libzstd.a
 libzstd.a:
 	$(MAKE) -C $(LIBDIR) libzstd.a-mt
 	cp $(LIBDIR)/libzstd.a .

tests/test-variants.sh

@@ -0,0 +1,115 @@
+#!/bin/sh
+set -e
+set -u
+set -x
+
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+PROG_DIR="$SCRIPT_DIR/../programs"
+
+ZSTD="$PROG_DIR/zstd"
+ZSTD_COMPRESS="$PROG_DIR/zstd-compress"
+ZSTD_DECOMPRESS="$PROG_DIR/zstd-decompress"
+ZSTD_NOLEGACY="$PROG_DIR/zstd-nolegacy"
+ZSTD_DICTBUILDER="$PROG_DIR/zstd-dictBuilder"
+ZSTD_FRUGAL="$PROG_DIR/zstd-frugal"
+ZSTD_NOMT="$PROG_DIR/zstd-nomt"
+
+println() {
+    printf '%b\n' "${*}"
+}
+
+die() {
+    println "$@" 1>&2
+    exit 1
+}
+
+symbol_present() {
+	(nm $1 || echo "symbol_present $@ failed") | grep $2
+}
+
+symbol_not_present() {
+	symbol_present $@ && die "Binary '$1' mistakenly contains symbol '$2'" ||:
+}
+
+compress_not_present() {
+	symbol_not_present "$1" ZSTD_compress
+}
+
+decompress_not_present() {
+	symbol_not_present "$1" ZSTD_decompress
+}
+
+dict_not_present() {
+	symbol_not_present "$1" ZDICT_
+	symbol_not_present "$1" COVER_
+}
+
+cliextra_not_present() {
+	symbol_not_present "$1" TRACE_
+	symbol_not_present "$1" BMK_
+}
+
+legacy_not_present() {
+	symbol_not_present "$1" ZSTDv0
+}
+
+test_help() {
+	"$1" --help | grep -- "$2"
+}
+
+test_no_help() {
+	test_help $@ && die "'$1' supports '$2' when it shouldn't" ||:
+}
+
+extras_not_present() {
+	dict_not_present $@
+	legacy_not_present $@
+	cliextra_not_present $@
+	test_no_help $@ "--train"
+	test_no_help $@ "-b#"
+}
+
+test_compress() {
+	echo "hello" | "$1" | "$ZSTD" -t
+}
+
+test_decompress() {
+	echo "hello" | "$ZSTD" | "$1" -t
+}
+
+test_zstd() {
+	test_compress $@
+	test_decompress $@
+}
+
+extras_not_present "$ZSTD_FRUGAL"
+extras_not_present "$ZSTD_COMPRESS"
+extras_not_present "$ZSTD_DECOMPRESS"
+
+compress_not_present "$ZSTD_DECOMPRESS"
+
+decompress_not_present "$ZSTD_COMPRESS"
+decompress_not_present "$ZSTD_DICTBUILDER"
+
+cliextra_not_present "$ZSTD_DICTBUILDER"
+
+legacy_not_present "$ZSTD_DICTBUILDER"
+legacy_not_present "$ZSTD_NOLEGACY"
+
+symbol_not_present "$ZSTD" ZSTDv01
+symbol_not_present "$ZSTD" ZSTDv02
+symbol_not_present "$ZSTD" ZSTDv03
+symbol_not_present "$ZSTD" ZSTDv04
+
+test_compress "$ZSTD_COMPRESS"
+test_decompress "$ZSTD_DECOMPRESS"
+
+test_zstd "$ZSTD_FRUGAL"
+test_zstd "$ZSTD_NOLEGACY"
+
+test_help "$ZSTD" '-b#'
+test_help "$ZSTD" --train
+test_help "$ZSTD_DICTBUILDER" --train
+
+println "Success!"