lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-29 11:21:22 +03:00
Code Activity

Change CLI to employ multithreading by default (#4211)

Browse Source 
* Change CLI to employ multithreading by default

* Document changes to benchmarking, print number of threads for display level >= 4, and add lower bound of 1 for the default number of threads
This commit is contained in:
daniellerozenblit
2024-12-12 13:09:29 -05:00
committed by GitHub
parent 709be6c227
commit 17beeb5d1a
3 changed files with 39 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
programs/zstdcli.c
View File

@ -8,22 +8,6 @@
* You may select, at your option, one of the above-listed licenses.
*/
/*-************************************
* Tuning parameters
**************************************/
#ifndef ZSTDCLI_CLEVEL_DEFAULT
# define ZSTDCLI_CLEVEL_DEFAULT 3
#endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
#endif
#ifndef ZSTDCLI_NBTHREADS_DEFAULT
# define ZSTDCLI_NBTHREADS_DEFAULT 1
#endif
/*-************************************
* Dependencies
**************************************/
@ -46,6 +30,23 @@
#endif
#include "../lib/zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
#include "fileio_asyncio.h"
#include "fileio_common.h"
/*-************************************
* Tuning parameters
**************************************/
#ifndef ZSTDCLI_CLEVEL_DEFAULT
# define ZSTDCLI_CLEVEL_DEFAULT 3
#endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
#endif
#ifndef ZSTDCLI_NBTHREADS_DEFAULT
#define ZSTDCLI_NBTHREADS_DEFAULT MAX(1, MIN(4, UTIL_countLogicalCores() / 4))
#endif
/*-************************************
@ -99,9 +100,7 @@ typedef enum { cover, fastCover, legacy } dictType;
/*-************************************
* Display Macros
**************************************/
#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__)
#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__)
#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__)
#undef DISPLAYLEVEL
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
@ -760,7 +759,7 @@ static int init_cLevel(void) {
}
#ifdef ZSTD_MULTITHREAD
static unsigned init_nbThreads(void) {
static unsigned default_nbThreads(void) {
const char* const env = getenv(ENV_NBTHREADS);
if (env != NULL) {
const char* ptr = env;
@ -855,7 +854,7 @@ int main(int argCount, const char* argv[])
ZSTD_paramSwitch_e mmapDict=ZSTD_ps_auto;
ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
FIO_compressionType_t cType = FIO_zstdCompression;
unsigned nbWorkers = 0;
int nbWorkers = -1; /* -1 means unset */
double compressibility = -1.0; /* lorem ipsum generator */
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
@ -896,7 +895,6 @@ int main(int argCount, const char* argv[])
#endif
ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto;
/* init */
checkLibVersion();
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
@ -904,9 +902,6 @@ int main(int argCount, const char* argv[])
assert(argCount >= 1);
if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); }
programName = lastNameFromPath(programName);
#ifdef ZSTD_MULTITHREAD
nbWorkers = init_nbThreads();
#endif
/* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
@ -1298,7 +1293,7 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, WELCOME_MESSAGE);
#ifdef ZSTD_MULTITHREAD
if ((operation==zom_decompress) && (!singleThread) && (nbWorkers > 1)) {
if ((operation==zom_decompress) && (nbWorkers > 1)) {
DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n");
}
if ((nbWorkers==0) && (!singleThread)) {
@ -1311,6 +1306,15 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
}
}
/* Resolve to default if nbWorkers is still unset */
if (nbWorkers == -1) {
if (operation == zom_decompress) {
nbWorkers = 1;
} else {
nbWorkers = default_nbThreads();
}
}
DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
#else
(void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
#endif