From 12e421b3c09aee78361a62c61f359fd8a5d930d3 Mon Sep 17 00:00:00 2001 From: Taylor Braun-Jones Date: Thu, 30 Jan 2020 10:31:00 -0500 Subject: [PATCH 1/4] Fix testing documentation typo --- TESTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TESTING.md b/TESTING.md index 42cd48e70..7e5305178 100644 --- a/TESTING.md +++ b/TESTING.md @@ -19,7 +19,7 @@ Medium tests run on every commit and pull request to `dev` branch, on TravisCI. They consist of the following tests: - The following tests run with UBsan and Asan on x86_64 and x86, as well as with Msan on x86_64 - - `tests/playTests.sh --test-long-data` + - `tests/playTests.sh --test-large-data` - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c` - `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode) - Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind) From 3cbc3d37e74c23354b4f10ab3fcf663ed43082cf Mon Sep 17 00:00:00 2001 From: Taylor Braun-Jones Date: Mon, 23 Mar 2020 17:46:56 -0400 Subject: [PATCH 2/4] Add documentation for -T option --- tests/fuzzer.c | 3 ++- tests/zstreamtest.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 3c4aac7a8..1b47a19e2 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -2833,7 +2833,8 @@ static int FUZ_usage(const char* programName) DISPLAY( " %s [args]\n", programName); DISPLAY( "\n"); DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%i) \n", nbTestsDefault); + DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault); + DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n"); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 890e66f9e..26a1d1a47 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -2373,7 +2373,8 @@ static int FUZ_usage(const char* programName) DISPLAY( " %s [args]\n", programName); DISPLAY( "\n"); DISPLAY( "Arguments :\n"); - DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); + DISPLAY( " -i# : Number of tests (default:%u)\n", nbTestsDefault); + DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n"); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); From 496d8c8b77fbfafd0ccf56a5d4a502e412070627 Mon Sep 17 00:00:00 2001 From: Taylor Braun-Jones Date: Mon, 23 Mar 2020 17:44:27 -0400 Subject: [PATCH 3/4] Respect the conventional BUILD_TESTS CMake option on the first configure --- build/cmake/CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index be5ce1518..70e8c0609 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -103,7 +103,14 @@ endif () option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON) option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF) -option(ZSTD_BUILD_TESTS "BUILD TESTS" OFF) + +# Respect the conventional CMake option for enabling tests if it was specified on the first configure +if (BUILD_TESTING) + set(ZSTD_BUILD_TESTS_default ON) +else() + set(ZSTD_BUILD_TESTS_default OFF) +endif() +option(ZSTD_BUILD_TESTS "BUILD TESTS" ${ZSTD_BUILD_TESTS_default}) if (MSVC) option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF) endif () From 5b054d9e904a50bf68417b5cb8fc888f2ed0aa24 Mon Sep 17 00:00:00 2001 From: Taylor Braun-Jones Date: Mon, 23 Mar 2020 16:08:52 -0400 Subject: [PATCH 4/4] Add support for running more tests via CTest --- .travis.yml | 2 +- Makefile | 2 +- appveyor.yml | 4 +-- build/cmake/tests/CMakeLists.txt | 50 ++++++++++++++++++++++++++++++-- 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 77227c364..e4e455602 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,7 +100,7 @@ matrix: - make clean - make -j check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" - - name: cmake build and test check # ~2mn + - name: cmake build and test check # ~6mn script: - make cmakebuild diff --git a/Makefile b/Makefile index 9b19d4d31..9300b1545 100644 --- a/Makefile +++ b/Makefile @@ -360,7 +360,7 @@ cmakebuild: $(MAKE) -C $(BUILDIR)/cmake/build -j4; $(MAKE) -C $(BUILDIR)/cmake/build install; $(MAKE) -C $(BUILDIR)/cmake/build uninstall; - cd $(BUILDIR)/cmake/build; ctest -V + cd $(BUILDIR)/cmake/build; ctest -V -L Medium c89build: clean $(CC) -v diff --git a/appveyor.yml b/appveyor.yml index 94e56330b..aae626d00 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -249,9 +249,9 @@ C:\cygwin64\bin\bash --login -c " set -e; cd build/cmake; - CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON .; + CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s .; make -j4; - ctest -V; + ctest -V -L Medium; " ) - if [%HOST%]==[mingw] ( diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt index 10039fb96..78cf32707 100644 --- a/build/cmake/tests/CMakeLists.txt +++ b/build/cmake/tests/CMakeLists.txt @@ -32,6 +32,16 @@ project(tests) +# name: Cache variable name. The value is expected to be a semicolon-separated +# list of command line flags +# default_value: Value to initialize the option with. Can be space separated. +function(AddTestFlagsOption name default_value doc) + string(STRIP "${default_value}" default_value) + string(REGEX REPLACE " +" ";" default_value "${default_value}") + set(${name} ${default_value} CACHE STRING "${doc}") + mark_as_advanced(${name}) +endfunction() + set(CMAKE_INCLUDE_CURRENT_DIR TRUE) # Define programs directory, where sources and header files are located @@ -43,18 +53,52 @@ include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/c add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c) target_link_libraries(datagen libzstd_static) +# +# fullbench +# add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c) target_link_libraries(fullbench libzstd_static) +add_test(NAME fullbench COMMAND fullbench) +# +# fuzzer +# add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c) target_link_libraries(fuzzer libzstd_static) +AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}" + "Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)") +add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS}) +# Disable the timeout since the run time is too long for the default timeout of +# 1500 seconds and varies considerably between low-end and high-end CPUs. +set_tests_properties(fuzzer PROPERTIES TIMEOUT 0) +# +# zstreamtest +# add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c) target_link_libraries(zstreamtest libzstd_static) +AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}" + "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)") +add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS}) -add_test( - NAME playTests - COMMAND sh -c "ZSTD_BIN='$' DATAGEN_BIN='$' '${TESTS_DIR}/playTests.sh'") +# +# playTests.sh +# +AddTestFlagsOption(ZSTD_PLAYTESTS_FLAGS "--test-large-data" + "Semicolon-separated list of flags to pass to the playTests.sh test") +add_test(NAME playTests COMMAND sh -c "${TESTS_DIR}/playTests.sh" ${ZSTD_PLAYTESTS_FLAGS}) +if (ZSTD_BUILD_PROGRAMS) + set_property(TEST playTests APPEND PROPERTY ENVIRONMENT + "ZSTD_BIN=$" + "DATAGEN_BIN=$" + ) +else() + message(STATUS "Disabling playTests.sh test because ZSTD_BUILD_PROGRAMS is not enabled") + set_tests_properties(playTests PROPERTIES DISABLED YES) +endif() + +# Label the "Medium" set of tests (see TESTING.md) +set_property(TEST fuzzer zstreamtest playTests APPEND PROPERTY LABELS Medium) if (UNIX) add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)