From 0665d4c1c2357fb7f90478994f8dedfb31dce164 Mon Sep 17 00:00:00 2001 From: Kevin Svetlitski Date: Fri, 5 Nov 2021 12:01:20 -0700 Subject: [PATCH 1/8] Display command line parameters with concrete values in verbose mode --- programs/fileio.c | 42 ++++++++++++++++++++++++++++++++++++++++++ programs/fileio.h | 1 + programs/zstdcli.c | 2 ++ 3 files changed, 45 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index 8e5301dee..8a6b98d27 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1782,6 +1782,48 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, return result; } +static const char* checked_index(const char* options[], size_t length, size_t index) { + assert(index < length); + return options[index]; +} + +#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (index)) + +void FIO_displayCompressionParameters(FIO_prefs_t* const prefs) { + static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION, + LZMA_EXTENSION, LZ4_EXTENSION}; + static const char* sparseOptions[3] = {" --no-sparse", "", " --sparse"}; + static const char* checkSumOptions[3] = {" --no-check", "", " --check"}; + static const char* rowMatchFinderOptions[3] = {"", " --no-row-match-finder", " --row-match-finder"}; + static const char* compressLiteralsOptions[3] = {"", " --compress-literals", " --no-compress-literals"}; + + assert(g_display_prefs.displayLevel >= 4); + + DISPLAY("--format=%s", formatOptions[prefs->compressionType]); + DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport)); + DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID"); + DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag)); + DISPLAY(" --block-size=%d", prefs->blockSize); + if (prefs->adaptiveMode) + DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel); + DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder)); + DISPLAY("%s", prefs->rsyncable ? " --rsyncable" : ""); + if (prefs->streamSrcSize) + DISPLAY(" --stream-size=%lu", prefs->streamSrcSize); + if (prefs->srcSizeHint) + DISPLAY(" --size-hint=%d", prefs->srcSizeHint); + if (prefs->targetCBlockSize) + DISPLAY(" --target-compressed-block-size=%lu", prefs->targetCBlockSize); + DISPLAY("%s", INDEX(compressLiteralsOptions, prefs->literalCompressionMode)); + DISPLAY(" --memory=%u", prefs->memLimit ? prefs->memLimit : 128 MB); + DISPLAY(" --threads=%d", prefs->nbWorkers); + DISPLAY("%s", prefs->excludeCompressedFiles ? " --exclude-compressed" : ""); + DISPLAY(" --%scontent-size", prefs->contentSize ? "" : "no-"); + DISPLAY("\n"); +} + +#undef INDEX + int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) diff --git a/programs/fileio.h b/programs/fileio.h index 82d5bc849..03527d042 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -108,6 +108,7 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices); void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); +void FIO_displayCompressionParameters(FIO_prefs_t* const prefs); /* FIO_ctx_t functions */ void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 8fb65111e..f895c4b23 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1430,6 +1430,8 @@ int main(int argCount, const char* argv[]) } } + if (g_displayLevel >= 4) + FIO_displayCompressionParameters(prefs); if ((filenames->tableSize==1) && outFileName) operationResult = FIO_compressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); else From 63fe6198ed9af6bf30de086238aa7e97009ac52a Mon Sep 17 00:00:00 2001 From: Kevin Svetlitski Date: Fri, 5 Nov 2021 12:48:13 -0700 Subject: [PATCH 2/8] Display --zstd= subparameters in command-line ready form in verbose mode --- programs/zstdcli.c | 62 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f895c4b23..8163ab37c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -647,6 +647,44 @@ static void printVersion(void) } } } +#define ZSTD_NB_STRATEGIES 9 +static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast", + "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", + "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; + +static void printDefaultCParams(const char* filename, const char* dictFileName, int cLevel) { + unsigned long long fileSize = UTIL_getFileSize(filename); + const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0; + const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize); + if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filename, (unsigned)fileSize); + else DISPLAY("%s (src size unknown)\n", filename); + DISPLAY(" - windowLog : %u\n", cParams.windowLog); + DISPLAY(" - chainLog : %u\n", cParams.chainLog); + DISPLAY(" - hashLog : %u\n", cParams.hashLog); + DISPLAY(" - searchLog : %u\n", cParams.searchLog); + DISPLAY(" - minMatch : %u\n", cParams.minMatch); + DISPLAY(" - targetLength : %u\n", cParams.targetLength); + assert(cParams.strategy < ZSTD_NB_STRATEGIES + 1); + DISPLAY(" - strategy : %s (%u)\n", ZSTD_strategyMap[(int)cParams.strategy], (unsigned)cParams.strategy); +} + +static void printActualCParams(const char* filename, const char* dictFileName, int cLevel, const ZSTD_compressionParameters* cParams) { + unsigned long long fileSize = UTIL_getFileSize(filename); + const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0; + ZSTD_compressionParameters actualCParams = ZSTD_getCParams(cLevel, fileSize, dictSize); + assert(g_displayLevel >= 4); + actualCParams.windowLog = cParams->windowLog == 0 ? actualCParams.windowLog : cParams->windowLog; + actualCParams.chainLog = cParams->chainLog == 0 ? actualCParams.chainLog : cParams->chainLog; + actualCParams.hashLog = cParams->hashLog == 0 ? actualCParams.hashLog : cParams->hashLog; + actualCParams.searchLog = cParams->searchLog == 0 ? actualCParams.searchLog : cParams->searchLog; + actualCParams.minMatch = cParams->minMatch == 0 ? actualCParams.minMatch : cParams->minMatch; + actualCParams.targetLength = cParams->targetLength == 0 ? actualCParams.targetLength : cParams->targetLength; + actualCParams.strategy = cParams->strategy == 0 ? actualCParams.strategy : cParams->strategy; + DISPLAY("--zstd=wlog=%d,clog=%d,hlog=%d,slog=%d,mml=%d,tlen=%d,strat=%d\n", + actualCParams.windowLog, actualCParams.chainLog, actualCParams.hashLog, actualCParams.searchLog, + actualCParams.minMatch, actualCParams.targetLength, actualCParams.strategy); +} + /* Environment variables for parameter setting */ #define ENV_CLEVEL "ZSTD_CLEVEL" #define ENV_NBTHREADS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */ @@ -723,11 +761,6 @@ static unsigned init_nbThreads(void) { val32 = readU32FromChar(&__nb); \ } -#define ZSTD_NB_STRATEGIES 9 -static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast", - "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", - "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; - typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode; #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -1411,22 +1444,13 @@ int main(int argCount, const char* argv[]) assert(ZSTD_NB_STRATEGIES == strategyBounds.upperBound); (void)strategyBounds; } - if (showDefaultCParams) { + if (showDefaultCParams || g_displayLevel >= 4) { size_t fileNb; for (fileNb = 0; fileNb < (size_t)filenames->tableSize; fileNb++) { - unsigned long long fileSize = UTIL_getFileSize(filenames->fileNames[fileNb]); - const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0; - const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize); - if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filenames->fileNames[fileNb], (unsigned)fileSize); - else DISPLAY("%s (src size unknown)\n", filenames->fileNames[fileNb]); - DISPLAY(" - windowLog : %u\n", cParams.windowLog); - DISPLAY(" - chainLog : %u\n", cParams.chainLog); - DISPLAY(" - hashLog : %u\n", cParams.hashLog); - DISPLAY(" - searchLog : %u\n", cParams.searchLog); - DISPLAY(" - minMatch : %u\n", cParams.minMatch); - DISPLAY(" - targetLength : %u\n", cParams.targetLength); - assert(cParams.strategy < ZSTD_NB_STRATEGIES + 1); - DISPLAY(" - strategy : %s (%u)\n", ZSTD_strategyMap[(int)cParams.strategy], (unsigned)cParams.strategy); + if (showDefaultCParams) + printDefaultCParams(filenames->fileNames[fileNb], dictFileName, cLevel); + if (g_displayLevel >= 4) + printActualCParams(filenames->fileNames[fileNb], dictFileName, cLevel, &compressionParams); } } From df9b7755cb38799e47c073fc5c03e146d23565fa Mon Sep 17 00:00:00 2001 From: Kevin Svetlitski Date: Wed, 10 Nov 2021 17:25:27 -0800 Subject: [PATCH 3/8] Fix const-ness of FIO_displayCompressionParameters --- programs/fileio.c | 2 +- programs/fileio.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 8a6b98d27..2f5850903 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1789,7 +1789,7 @@ static const char* checked_index(const char* options[], size_t length, size_t in #define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (index)) -void FIO_displayCompressionParameters(FIO_prefs_t* const prefs) { +void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) { static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION, LZMA_EXTENSION, LZ4_EXTENSION}; static const char* sparseOptions[3] = {" --no-sparse", "", " --sparse"}; diff --git a/programs/fileio.h b/programs/fileio.h index 03527d042..61094db83 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -108,7 +108,7 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices); void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); -void FIO_displayCompressionParameters(FIO_prefs_t* const prefs); +void FIO_displayCompressionParameters(const FIO_prefs_t* prefs); /* FIO_ctx_t functions */ void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); From f6ffd392302df82e348a1ab42db3c1374bb29605 Mon Sep 17 00:00:00 2001 From: Kevin Svetlitski Date: Thu, 11 Nov 2021 11:57:55 -0800 Subject: [PATCH 4/8] Add test case for detailed compression parameter verbose output --- tests/test_verbose_output.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 tests/test_verbose_output.sh diff --git a/tests/test_verbose_output.sh b/tests/test_verbose_output.sh new file mode 100755 index 000000000..9fc8a29e9 --- /dev/null +++ b/tests/test_verbose_output.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e -u -o pipefail + +make -j -C .. CFLAGS=-O1 zstd +empty_file=$(mktemp) +trap 'rm -f $empty_file' EXIT +../zstd -vv "$empty_file" 2>&1 | \ +grep -q -E -- "--zstd=wlog=[[:digit:]]+,clog=[[:digit:]]+,hlog=[[:digit:]]+,\ +slog=[[:digit:]]+,mml=[[:digit:]]+,tlen=[[:digit:]]+,strat=[[:digit:]]+" From 365c91194ce308650197fa08ecee9d802f7225a1 Mon Sep 17 00:00:00 2001 From: Kevin Svetlitski Date: Thu, 11 Nov 2021 12:14:56 -0800 Subject: [PATCH 5/8] Ensure print*CParams functions are only defined when used --- programs/zstdcli.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 8163ab37c..05c537b5e 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -652,6 +652,8 @@ static const char* ZSTD_strategyMap[ZSTD_NB_STRATEGIES + 1] = { "", "ZSTD_fast", "ZSTD_dfast", "ZSTD_greedy", "ZSTD_lazy", "ZSTD_lazy2", "ZSTD_btlazy2", "ZSTD_btopt", "ZSTD_btultra", "ZSTD_btultra2"}; +#ifndef ZSTD_NOCOMPRESS + static void printDefaultCParams(const char* filename, const char* dictFileName, int cLevel) { unsigned long long fileSize = UTIL_getFileSize(filename); const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0; @@ -685,6 +687,8 @@ static void printActualCParams(const char* filename, const char* dictFileName, i actualCParams.minMatch, actualCParams.targetLength, actualCParams.strategy); } +#endif + /* Environment variables for parameter setting */ #define ENV_CLEVEL "ZSTD_CLEVEL" #define ENV_NBTHREADS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */ From 375e3aad6c35be2605acbec873d8ad6c54f49434 Mon Sep 17 00:00:00 2001 From: Kevin Svetlitski Date: Thu, 11 Nov 2021 13:17:30 -0800 Subject: [PATCH 6/8] Ensure formatting directives for displaying size_t are portable --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 2f5850903..cf3e7414c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1809,11 +1809,11 @@ void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) { DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder)); DISPLAY("%s", prefs->rsyncable ? " --rsyncable" : ""); if (prefs->streamSrcSize) - DISPLAY(" --stream-size=%lu", prefs->streamSrcSize); + DISPLAY(" --stream-size=%u", (unsigned) prefs->streamSrcSize); if (prefs->srcSizeHint) DISPLAY(" --size-hint=%d", prefs->srcSizeHint); if (prefs->targetCBlockSize) - DISPLAY(" --target-compressed-block-size=%lu", prefs->targetCBlockSize); + DISPLAY(" --target-compressed-block-size=%u", (unsigned) prefs->targetCBlockSize); DISPLAY("%s", INDEX(compressLiteralsOptions, prefs->literalCompressionMode)); DISPLAY(" --memory=%u", prefs->memLimit ? prefs->memLimit : 128 MB); DISPLAY(" --threads=%d", prefs->nbWorkers); From 7fbd126e089a1359ce7c9729be91ee546312af1a Mon Sep 17 00:00:00 2001 From: Kevin Svetlitski Date: Thu, 11 Nov 2021 14:37:02 -0800 Subject: [PATCH 7/8] Suppress spurious unused parameter warning --- programs/fileio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index cf3e7414c..c1881a4f0 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1784,6 +1784,8 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, static const char* checked_index(const char* options[], size_t length, size_t index) { assert(index < length); + // Necessary to avoid warnings since -O3 will omit the above `assert` + (void) length; return options[index]; } From 9b28c26cbff0d43eb72d90ba5ef2fea27659638e Mon Sep 17 00:00:00 2001 From: Kevin Svetlitski Date: Fri, 12 Nov 2021 14:10:21 -0800 Subject: [PATCH 8/8] Integrate verbose mode tests into playTests.sh --- tests/playTests.sh | 7 +++++++ tests/test_verbose_output.sh | 9 --------- 2 files changed, 7 insertions(+), 9 deletions(-) delete mode 100755 tests/test_verbose_output.sh diff --git a/tests/playTests.sh b/tests/playTests.sh index 69653276a..5d26fb8f5 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -753,6 +753,13 @@ datagen -g257000 > tmp_files/tmp3 zstd --show-default-cparams -f -r tmp_files rm -rf tmp* +println "test : show compression parameters in verbose mode" +datagen > tmp +zstd -vv tmp 2>&1 | \ +grep -q -E -- "--zstd=wlog=[[:digit:]]+,clog=[[:digit:]]+,hlog=[[:digit:]]+,\ +slog=[[:digit:]]+,mml=[[:digit:]]+,tlen=[[:digit:]]+,strat=[[:digit:]]+" +rm -rf tmp* + println "\n===> Advanced compression parameters " println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" diff --git a/tests/test_verbose_output.sh b/tests/test_verbose_output.sh deleted file mode 100755 index 9fc8a29e9..000000000 --- a/tests/test_verbose_output.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -e -u -o pipefail - -make -j -C .. CFLAGS=-O1 zstd -empty_file=$(mktemp) -trap 'rm -f $empty_file' EXIT -../zstd -vv "$empty_file" 2>&1 | \ -grep -q -E -- "--zstd=wlog=[[:digit:]]+,clog=[[:digit:]]+,hlog=[[:digit:]]+,\ -slog=[[:digit:]]+,mml=[[:digit:]]+,tlen=[[:digit:]]+,strat=[[:digit:]]+"