From d898fb7ba60611a3c293c8b8098960263e0bd7db Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 17 Nov 2017 00:22:55 -0800 Subject: [PATCH] bench: added cli command `-S` to benchmark multiple files separately Currently, all files are joined by default, they are compressed separately but benchmarked together, providing a single final result. Benchmarking files separately make it possible to accurately measure difference for each file. This is expected to be useful while tuning optimal parser. --- programs/bench.c | 47 ++++++++++++++++++++++++++++++++-------------- programs/bench.h | 1 + programs/util.h | 2 +- programs/zstdcli.c | 8 ++++++++ 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index c61273936..e28bed747 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -135,6 +135,11 @@ void BMK_setRealTime(unsigned priority) { g_realTime = (priority>0); } +static U32 g_separateFiles = 0; +void BMK_setSeparateFiles(unsigned separate) { + g_separateFiles = (separate>0); +} + static U32 g_ldmFlag = 0; void BMK_setLdmFlag(unsigned ldmFlag) { g_ldmFlag = ldmFlag; @@ -482,11 +487,11 @@ static size_t BMK_findMaxMem(U64 requiredMem) return (size_t)(requiredMem); } -static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, +static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize, - ZSTD_compressionParameters *compressionParams) + const ZSTD_compressionParameters* const compressionParams) { int l; @@ -518,7 +523,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, At most, fills `buffer` entirely */ static void BMK_loadFiles(void* buffer, size_t bufferSize, size_t* fileSizes, - const char** fileNamesTable, unsigned nbFiles) + const char* const * const fileNamesTable, unsigned nbFiles) { size_t pos = 0, totalSize = 0; unsigned n; @@ -550,16 +555,17 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, - int cLevelLast, ZSTD_compressionParameters *compressionParams) +static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles, + const char* const dictFileName, + int const cLevel, int const cLevelLast, + const ZSTD_compressionParameters* const compressionParams) { void* srcBuffer; size_t benchedSize; void* dictBuffer = NULL; size_t dictBufferSize = 0; - size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); - char mfName[20] = {0}; if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); @@ -588,13 +594,26 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); /* Bench */ - snprintf (mfName, sizeof(mfName), " %u files", nbFiles); - { const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; - BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, cLevelLast, - fileSizes, nbFiles, - dictBuffer, dictBufferSize, compressionParams); - } + if (g_separateFiles) { + const BYTE* srcPtr = (const BYTE*)srcBuffer; + U32 fileNb; + for (fileNb=0; fileNb 1) ? mfName : fileNamesTable[0]; + BMK_benchCLevel(srcBuffer, benchedSize, + displayName, cLevel, cLevelLast, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, compressionParams); + } } /* clean up */ free(srcBuffer); diff --git a/programs/bench.h b/programs/bench.h index f4d415488..597f25036 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -25,6 +25,7 @@ void BMK_setBlockSize(size_t blockSize); void BMK_setNbThreads(unsigned nbThreads); void BMK_setRealTime(unsigned priority); void BMK_setNotificationLevel(unsigned level); +void BMK_setSeparateFiles(unsigned separate); void BMK_setAdditionalParam(int additionalParam); void BMK_setDecodeOnlyMode(unsigned decodeFlag); void BMK_setLdmFlag(unsigned ldmFlag); diff --git a/programs/util.h b/programs/util.h index c5e4365d0..e44d7459d 100644 --- a/programs/util.h +++ b/programs/util.h @@ -336,7 +336,7 @@ UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) } -UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) +UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles) { U64 total = 0; int error = 0; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d5cb6285c..91af49e24 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -377,6 +377,7 @@ int main(int argCount, const char* argv[]) lastCommand = 0, nbThreads = 1, setRealTimePrio = 0, + separateFiles = 0, ldmFlag = 0; unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; @@ -633,6 +634,12 @@ int main(int argCount, const char* argv[]) blockSize = readU32FromChar(&argument); break; + /* benchmark files separately (hidden option) */ + case 'S': + argument++; + separateFiles = 1; + break; + #endif /* ZSTD_NOBENCH */ /* nb of threads (hidden option) */ @@ -751,6 +758,7 @@ int main(int argCount, const char* argv[]) if (operation==zom_bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(g_displayLevel); + BMK_setSeparateFiles(separateFiles); BMK_setBlockSize(blockSize); BMK_setNbThreads(nbThreads); BMK_setRealTime(setRealTimePrio);