Change CLI to employ multithreading by default (#4211)

* Change CLI to employ multithreading by default

* Document changes to benchmarking, print number of threads for display level >= 4, and add lower bound of 1 for the default number of threads

programs/fileio_common.h

@@ -28,11 +28,14 @@ extern "C" {
 #define GB *(1U<<30)
 #undef MAX
 #define MAX(a,b) ((a)>(b) ? (a) : (b))
+#undef MIN  /* in case it would be already defined */
+#define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
 extern FIO_display_prefs_t g_display_prefs;
 
-#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAY_F(f, ...)    fprintf((f), __VA_ARGS__)
-#define DISPLAYOUT(...)      fprintf(stdout, __VA_ARGS__)
+#define DISPLAYOUT(...)      DISPLAY_F(stdout, __VA_ARGS__)
+#define DISPLAY(...)         DISPLAY_F(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
 
 extern UTIL_time_t g_displayClock;
@@ -56,10 +59,6 @@ extern UTIL_time_t g_displayClock;
 #define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
 #define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }
 
-#undef MIN  /* in case it would be already defined */
-#define MIN(a,b)    ((a) < (b) ? (a) : (b))
-
-
 #define EXM_THROW(error, ...)                                             \
 {                                                                         \
     DISPLAYLEVEL(1, "zstd: ");                                            \

programs/zstd.1.md

@@ -343,7 +343,7 @@ If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a
 
 `ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression.
 If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message.
-`ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200.
+`ZSTD_NBTHREADS` has a default value of `max(1, min(4, nbCores/4))`, and is capped at ZSTDMT_NBWORKERS_MAX==200.
 `zstd` must be compiled with multithread support for this variable to have any effect.
 
 They can both be overridden by corresponding command line arguments:
@@ -664,9 +664,12 @@ BENCHMARK
 The `zstd` CLI provides a benchmarking mode that can be used to easily find suitable compression parameters, or alternatively to benchmark a computer's performance.
 `zstd -b [FILE(s)]` will benchmark `zstd` for both compression and decompression using default compression level.
 Note that results are very dependent on the content being compressed.
+
 It's possible to pass multiple files to the benchmark, and even a directory with `-r DIRECTORY`.
 When no `FILE` is provided, the benchmark will use a procedurally generated `lorem ipsum` text.
 
+Benchmarking will employ `max(1, min(4, nbCores/4))` worker threads by default in order to match the behavior of the normal CLI I/O.
+
 * `-b#`:
     benchmark file(s) using compression level #
 * `-e#`:

programs/zstdcli.c

@@ -8,22 +8,6 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-
-/*-************************************
-*  Tuning parameters
-**************************************/
-#ifndef ZSTDCLI_CLEVEL_DEFAULT
-#  define ZSTDCLI_CLEVEL_DEFAULT 3
-#endif
-
-#ifndef ZSTDCLI_CLEVEL_MAX
-#  define ZSTDCLI_CLEVEL_MAX 19   /* without using --ultra */
-#endif
-
-#ifndef ZSTDCLI_NBTHREADS_DEFAULT
-#  define ZSTDCLI_NBTHREADS_DEFAULT 1
-#endif
-
 /*-************************************
 *  Dependencies
 **************************************/
@@ -46,6 +30,23 @@
 #endif
 #include "../lib/zstd.h"  /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
 #include "fileio_asyncio.h"
+#include "fileio_common.h"
+
+/*-************************************
+*  Tuning parameters
+**************************************/
+#ifndef ZSTDCLI_CLEVEL_DEFAULT
+#  define ZSTDCLI_CLEVEL_DEFAULT 3
+#endif
+
+#ifndef ZSTDCLI_CLEVEL_MAX
+#  define ZSTDCLI_CLEVEL_MAX 19   /* without using --ultra */
+#endif
+
+#ifndef ZSTDCLI_NBTHREADS_DEFAULT
+#define ZSTDCLI_NBTHREADS_DEFAULT MAX(1, MIN(4, UTIL_countLogicalCores() / 4))
+#endif
+
 
 
 /*-************************************
@@ -99,9 +100,7 @@ typedef enum { cover, fastCover, legacy } dictType;
 /*-************************************
 *  Display Macros
 **************************************/
-#define DISPLAY_F(f, ...)    fprintf((f), __VA_ARGS__)
+#undef DISPLAYLEVEL
-#define DISPLAYOUT(...)      DISPLAY_F(stdout, __VA_ARGS__)
-#define DISPLAY(...)         DISPLAY_F(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
 static int g_displayLevel = DISPLAY_LEVEL_DEFAULT;   /* 0 : no display,  1: errors,  2 : + result + interaction + warnings,  3 : + progression,  4 : + information */
 
@@ -760,7 +759,7 @@ static int init_cLevel(void) {
 }
 
 #ifdef ZSTD_MULTITHREAD
-static unsigned init_nbThreads(void) {
+static unsigned default_nbThreads(void) {
     const char* const env = getenv(ENV_NBTHREADS);
     if (env != NULL) {
         const char* ptr = env;
@@ -855,7 +854,7 @@ int main(int argCount, const char* argv[])
     ZSTD_paramSwitch_e mmapDict=ZSTD_ps_auto;
     ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
     FIO_compressionType_t cType = FIO_zstdCompression;
-    unsigned nbWorkers = 0;
+    int nbWorkers = -1; /* -1 means unset */
     double compressibility = -1.0;  /* lorem ipsum generator */
     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
     size_t blockSize = 0;
@@ -896,7 +895,6 @@ int main(int argCount, const char* argv[])
 #endif
     ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto;
 
-
     /* init */
     checkLibVersion();
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
@@ -904,9 +902,6 @@ int main(int argCount, const char* argv[])
     assert(argCount >= 1);
     if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); }
     programName = lastNameFromPath(programName);
-#ifdef ZSTD_MULTITHREAD
-    nbWorkers = init_nbThreads();
-#endif
 
     /* preset behaviors */
     if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
@@ -1298,7 +1293,7 @@ int main(int argCount, const char* argv[])
     DISPLAYLEVEL(3, WELCOME_MESSAGE);
 
 #ifdef ZSTD_MULTITHREAD
-    if ((operation==zom_decompress) && (!singleThread) && (nbWorkers > 1)) {
+    if ((operation==zom_decompress) && (nbWorkers > 1)) {
         DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n");
     }
     if ((nbWorkers==0) && (!singleThread)) {
@@ -1311,6 +1306,15 @@ int main(int argCount, const char* argv[])
             DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
         }
     }
+    /* Resolve to default if nbWorkers is still unset */
+    if (nbWorkers == -1) {
+      if (operation == zom_decompress) {
+        nbWorkers = 1;
+      } else {
+        nbWorkers = default_nbThreads();
+      }
+    }
+    DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
 #else
     (void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
 #endif