From 13cb7a10ae556c24896ee2f305624d084234042a Mon Sep 17 00:00:00 2001 From: Dave Vasilevsky Date: Wed, 7 May 2025 01:43:36 -0400 Subject: [PATCH] seekable_format: Add test for parallel_compression memory usage Use ulimit to fail the test if we use O(filesize) memory, rather than O(threads). --- contrib/seekable_format/tests/.gitignore | 1 + contrib/seekable_format/tests/Makefile | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/contrib/seekable_format/tests/.gitignore b/contrib/seekable_format/tests/.gitignore index f831eaf37..b59d08b34 100644 --- a/contrib/seekable_format/tests/.gitignore +++ b/contrib/seekable_format/tests/.gitignore @@ -1 +1,2 @@ seekable_tests +data.txt diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile index a81f2229f..a23c0f43c 100644 --- a/contrib/seekable_format/tests/Makefile +++ b/contrib/seekable_format/tests/Makefile @@ -24,7 +24,7 @@ SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB) .PHONY: default clean test default: test -test: seekable_tests +test: seekable_tests parallel_compression_test ./seekable_tests $(ZSTDLIB): @@ -32,7 +32,27 @@ $(ZSTDLIB): seekable_tests : $(SEEKABLE_OBJS) +EXAMPLES_PATH = ../examples +PARALLEL_COMPRESSION = $(EXAMPLES_PATH)/parallel_compression + +DATAGEN_PATH = ../../../tests +DATAGEN = $(DATAGEN_PATH)/datagen + +$(PARALLEL_COMPRESSION): + $(MAKE) -C $(EXAMPLES_PATH) parallel_compression + +$(DATAGEN): + $(MAKE) -C $(DATAGEN_PATH) datagen + +data.txt: $(DATAGEN) + $(DATAGEN) -g100M > $@ + +parallel_compression_test: $(PARALLEL_COMPRESSION) data.txt + ulimit -Sv 102400; $(PARALLEL_COMPRESSION) data.txt 1048576 2 + +.PHONY: parallel_compression_test parallel_comp + clean: @$(RM) core *.o tmp* result* *.zst \ - seekable_tests + seekable_tests data.txt @echo Cleaning completed