From c78f434aa4f5f1097c8edb975f4c1635817a5a71 Mon Sep 17 00:00:00 2001 From: Yonatan Komornik <11005061+yoniko@users.noreply.github.com> Date: Sun, 12 Feb 2023 12:32:31 -0800 Subject: [PATCH] Fix zstd-dll build missing dependencies (#3496) * Fixes zstd-dll build (https://github.com/facebook/zstd/issues/3492): - Adds pool.o and threading.o dependency to the zstd-dll target - Moves custom allocation functions into header to avoid needing to add dependency on common.o - Adds test target for zstd-dll - Adds github workflow that buildis zstd-dll --- .github/workflows/dev-short-tests.yml | 9 +++- contrib/linux-kernel/zstd_common_module.c | 3 -- lib/common/allocations.h | 55 +++++++++++++++++++++++ lib/common/pool.c | 2 +- lib/common/zstd_common.c | 35 --------------- lib/common/zstd_internal.h | 5 --- lib/compress/zstd_compress.c | 1 + lib/compress/zstd_cwksp.h | 1 + lib/compress/zstdmt_compress.c | 1 + lib/decompress/zstd_ddict.c | 1 + lib/decompress/zstd_decompress.c | 1 + programs/Makefile | 2 +- tests/Makefile | 8 +++- 13 files changed, 76 insertions(+), 48 deletions(-) create mode 100644 lib/common/allocations.h diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml index 655394f84..092c933c7 100644 --- a/.github/workflows/dev-short-tests.yml +++ b/.github/workflows/dev-short-tests.yml @@ -55,6 +55,14 @@ jobs: run: | make c89build V=1 + build-zstd-dll: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3 + - name: build zstd bin against a dynamic lib (debuglevel for more dependencies) + run: | + make -C lib lib-mt-release + DEBUGLEVEL=2 make -C programs zstd-dll gcc-7-libzstd: runs-on: ubuntu-latest @@ -339,7 +347,6 @@ jobs: make -j -C programs allVariants MOREFLAGS=-O0 ./tests/test-variants.sh - qemu-consistency: name: QEMU ${{ matrix.name }} runs-on: ubuntu-20.04 diff --git a/contrib/linux-kernel/zstd_common_module.c b/contrib/linux-kernel/zstd_common_module.c index 2fead39eb..466828e35 100644 --- a/contrib/linux-kernel/zstd_common_module.c +++ b/contrib/linux-kernel/zstd_common_module.c @@ -24,9 +24,6 @@ EXPORT_SYMBOL_GPL(HUF_readStats_wksp); EXPORT_SYMBOL_GPL(ZSTD_isError); EXPORT_SYMBOL_GPL(ZSTD_getErrorName); EXPORT_SYMBOL_GPL(ZSTD_getErrorCode); -EXPORT_SYMBOL_GPL(ZSTD_customMalloc); -EXPORT_SYMBOL_GPL(ZSTD_customCalloc); -EXPORT_SYMBOL_GPL(ZSTD_customFree); MODULE_LICENSE("Dual BSD/GPL"); MODULE_DESCRIPTION("Zstd Common"); diff --git a/lib/common/allocations.h b/lib/common/allocations.h new file mode 100644 index 000000000..a3153c4ba --- /dev/null +++ b/lib/common/allocations.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides custom allocation primitives + */ + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ + +#include "mem.h" /* MEM_STATIC */ +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" /* ZSTD_customMem */ + +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H + +/* custom memory allocation functions */ + +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} + +#endif /* ZSTD_ALLOCATIONS_H */ diff --git a/lib/common/pool.c b/lib/common/pool.c index f3d9d0854..d5ca5a780 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -10,9 +10,9 @@ /* ====== Dependencies ======= */ +#include "../common/allocations.h" /* ZSTD_customCalloc, ZSTD_customFree */ #include "zstd_deps.h" /* size_t */ #include "debug.h" /* assert */ -#include "zstd_internal.h" /* ZSTD_customCalloc, ZSTD_customFree */ #include "pool.h" /* ====== Compiler specifics ====== */ diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 320855247..3f04c22ab 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -14,7 +14,6 @@ * Dependencies ***************************************/ #define ZSTD_DEPS_NEED_MALLOC -#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ #include "error_private.h" #include "zstd_internal.h" @@ -47,37 +46,3 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } /*! ZSTD_getErrorString() : * provides error code string from enum */ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } - - - -/*=************************************************************** -* Custom allocator -****************************************************************/ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) - return customMem.customAlloc(customMem.opaque, size); - return ZSTD_malloc(size); -} - -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) { - /* calloc implemented as malloc+memset; - * not as efficient as calloc, but next best guess for custom malloc */ - void* const ptr = customMem.customAlloc(customMem.opaque, size); - ZSTD_memset(ptr, 0, size); - return ptr; - } - return ZSTD_calloc(1, size); -} - -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) -{ - if (ptr!=NULL) { - if (customMem.customFree) - customMem.customFree(customMem.opaque, ptr); - else - ZSTD_free(ptr); - } -} diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 37836dc70..1f942f27b 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -350,11 +350,6 @@ typedef struct { const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ -/* custom memory allocation functions */ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem); -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem); - /* ZSTD_invalidateRepCodes() : * ensures next compression will not use repcodes from previous block. diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b55f684cd..173ce2421 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -11,6 +11,7 @@ /*-************************************* * Dependencies ***************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ #include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ #include "../common/mem.h" #include "hist.h" /* HIST_countFast_wksp */ diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 97676693b..9113cdbd4 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -14,6 +14,7 @@ /*-************************************* * Dependencies ***************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ #include "../common/zstd_internal.h" #if defined (__cplusplus) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 7a2c71720..bf2de4a79 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -20,6 +20,7 @@ /* ====== Dependencies ====== */ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */ #include "../common/mem.h" /* MEM_STATIC */ #include "../common/pool.h" /* threadpool */ diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c index ad5c34a7f..309ec0d03 100644 --- a/lib/decompress/zstd_ddict.c +++ b/lib/decompress/zstd_ddict.c @@ -14,6 +14,7 @@ /*-******************************************************* * Dependencies *********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ #include "../common/cpu.h" /* bmi2 */ #include "../common/mem.h" /* low level memory routines */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d487966cc..3f3ca57c0 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -55,6 +55,7 @@ /*-******************************************************* * Dependencies *********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ #include "../common/mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY diff --git a/programs/Makefile b/programs/Makefile index fcff41dc1..8507abef3 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -225,7 +225,7 @@ zstd-noxz : zstd .PHONY: zstd-dll zstd-dll : LDFLAGS+= -L$(LIBZSTD) zstd-dll : LDLIBS += -lzstd -zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c +zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c pool.c threading.c zstd-dll : zstd diff --git a/tests/Makefile b/tests/Makefile index 3eed19ea8..778c7d675 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -95,7 +95,7 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus dll: fuzzer-dll zstreamtest-dll .PHONY: zstd zstd32 zstd-nolegacy # only external makefile knows how to build or update them -zstd zstd32 zstd-nolegacy: +zstd zstd32 zstd-nolegacy zstd-dll: $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" .PHONY: libzstd @@ -328,13 +328,17 @@ test-all: test test32 test-decodecorpus-cli test-zstd: ZSTD = $(PRGDIR)/zstd test-zstd: zstd +.PHONY: test-zstd-dll +test-zstd-dll: ZSTD = $(PRGDIR)/zstd +test-zstd-dll: zstd-dll + test-zstd32: ZSTD = $(PRGDIR)/zstd32 test-zstd32: zstd32 test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy test-zstd-nolegacy: zstd-nolegacy -test-zstd test-zstd32 test-zstd-nolegacy: datagen +test-zstd test-zstd32 test-zstd-nolegacy test-zstd-dll: datagen file $(ZSTD) EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)