lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-05 19:15:58 +03:00
Code Activity

Change CLI to employ multithreading by default (#4211)

Browse Source 
* Change CLI to employ multithreading by default

* Document changes to benchmarking, print number of threads for display level >= 4, and add lower bound of 1 for the default number of threads
This commit is contained in:
daniellerozenblit
2024-12-12 13:09:29 -05:00
committed by GitHub
parent 709be6c227
commit 17beeb5d1a
3 changed files with 39 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
programs/fileio_common.h
View File

@@ -28,11 +28,14 @@ extern "C" {
#define GB *(1U<<30)
#undef MAX
#define MAX(a,b) ((a)>(b) ? (a) : (b))
#undef MIN /* in case it would be already defined */
#define MIN(a,b) ((a) < (b) ? (a) : (b))
extern FIO_display_prefs_t g_display_prefs;
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__)
#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__)
#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
extern UTIL_time_t g_displayClock;
@@ -56,10 +59,6 @@ extern UTIL_time_t g_displayClock;
#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }
#undef MIN /* in case it would be already defined */
#define MIN(a,b) ((a) < (b) ? (a) : (b))
#define EXM_THROW(error, ...) \
{ \
DISPLAYLEVEL(1, "zstd: "); \

5
programs/zstd.1.md
View File

@@ -343,7 +343,7 @@ If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a
`ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression.
If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message.
`ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200.
`ZSTD_NBTHREADS` has a default value of `max(1, min(4, nbCores/4))`, and is capped at ZSTDMT_NBWORKERS_MAX==200.
`zstd` must be compiled with multithread support for this variable to have any effect.
They can both be overridden by corresponding command line arguments:
@@ -664,9 +664,12 @@ BENCHMARK
The `zstd` CLI provides a benchmarking mode that can be used to easily find suitable compression parameters, or alternatively to benchmark a computer's performance.
`zstd -b [FILE(s)]` will benchmark `zstd` for both compression and decompression using default compression level.
Note that results are very dependent on the content being compressed.
It's possible to pass multiple files to the benchmark, and even a directory with `-r DIRECTORY`.
When no `FILE` is provided, the benchmark will use a procedurally generated `lorem ipsum` text.
Benchmarking will employ `max(1, min(4, nbCores/4))` worker threads by default in order to match the behavior of the normal CLI I/O.
* `-b#`:
benchmark file(s) using compression level #
* `-e#`:

56
programs/zstdcli.c
View File

@@ -8,22 +8,6 @@
* You may select, at your option, one of the above-listed licenses.
*/
/*-************************************
* Tuning parameters
**************************************/
#ifndef ZSTDCLI_CLEVEL_DEFAULT
# define ZSTDCLI_CLEVEL_DEFAULT 3
#endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
#endif
#ifndef ZSTDCLI_NBTHREADS_DEFAULT
# define ZSTDCLI_NBTHREADS_DEFAULT 1
#endif
/*-************************************
* Dependencies
**************************************/
@@ -46,6 +30,23 @@
#endif
#include "../lib/zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
#include "fileio_asyncio.h"
#include "fileio_common.h"
/*-************************************
* Tuning parameters
**************************************/
#ifndef ZSTDCLI_CLEVEL_DEFAULT
# define ZSTDCLI_CLEVEL_DEFAULT 3
#endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
#endif
#ifndef ZSTDCLI_NBTHREADS_DEFAULT
#define ZSTDCLI_NBTHREADS_DEFAULT MAX(1, MIN(4, UTIL_countLogicalCores() / 4))
#endif
/*-************************************
@@ -99,9 +100,7 @@ typedef enum { cover, fastCover, legacy } dictType;
/*-************************************
* Display Macros
**************************************/
#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__)
#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__)
#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__)
#undef DISPLAYLEVEL
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
@@ -760,7 +759,7 @@ static int init_cLevel(void) {
}
#ifdef ZSTD_MULTITHREAD
static unsigned init_nbThreads(void) {
static unsigned default_nbThreads(void) {
const char* const env = getenv(ENV_NBTHREADS);
if (env != NULL) {
const char* ptr = env;
@@ -855,7 +854,7 @@ int main(int argCount, const char* argv[])
ZSTD_paramSwitch_e mmapDict=ZSTD_ps_auto;
ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
FIO_compressionType_t cType = FIO_zstdCompression;
unsigned nbWorkers = 0;
int nbWorkers = -1; /* -1 means unset */
double compressibility = -1.0; /* lorem ipsum generator */
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
@@ -896,7 +895,6 @@ int main(int argCount, const char* argv[])
#endif
ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto;
/* init */
checkLibVersion();
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
@@ -904,9 +902,6 @@ int main(int argCount, const char* argv[])
assert(argCount >= 1);
if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); }
programName = lastNameFromPath(programName);
#ifdef ZSTD_MULTITHREAD
nbWorkers = init_nbThreads();
#endif
/* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
@@ -1298,7 +1293,7 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, WELCOME_MESSAGE);
#ifdef ZSTD_MULTITHREAD
if ((operation==zom_decompress) && (!singleThread) && (nbWorkers > 1)) {
if ((operation==zom_decompress) && (nbWorkers > 1)) {
DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n");
}
if ((nbWorkers==0) && (!singleThread)) {
@@ -1311,6 +1306,15 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
}
}
/* Resolve to default if nbWorkers is still unset */
if (nbWorkers == -1) {
if (operation == zom_decompress) {
nbWorkers = 1;
} else {
nbWorkers = default_nbThreads();
}
}
DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
#else
(void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
#endif