From 805c5a3efbc22ef7735f009730aff8c3f95b6b86 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 1 Apr 2017 00:36:31 -0700 Subject: [PATCH] updated documentation on multithreading modes --- programs/zstd.1 | 20 +++++++++++++++++--- programs/zstd.1.md | 30 ++++++++++++++++++++++++++++-- programs/zstdcli.c | 16 ++++++++-------- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/programs/zstd.1 b/programs/zstd.1 index 8b418a0ee..3b0064993 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -1,5 +1,5 @@ . -.TH "ZSTD" "1" "March 2017" "zstd 1.1.5" "User Commands" +.TH "ZSTD" "1" "April 2017" "zstd 1.1.5" "User Commands" . .SH "NAME" \fBzstd\fR \- zstd, unzstd, zstdcat \- Compress or decompress \.zst files @@ -97,6 +97,10 @@ Use FILEs as a training set to create a dictionary\. The training set should con unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\. . .TP +\fB\-T#\fR +Compress using # threads (default: 1)\. This modifier is only available if \fBzstd\fR was compiled with multithreading support\. +. +.TP \fB\-D file\fR use \fBfile\fR as Dictionary to compress or decompress FILE(s) . @@ -126,7 +130,7 @@ remove source file(s) after successful compression or decompression . .TP \fB\-k\fR, \fB\-\-keep\fR -keep source file(s) after successful compression or decompression\. This is the default behaviour\. +keep source file(s) after successful compression or decompression\. This is the default behavior\. . .TP \fB\-r\fR @@ -150,7 +154,7 @@ suppress warnings, interactivity, and notifications\. specify twice to suppress . .TP \fB\-C\fR, \fB\-\-[no\-]check\fR -add integrety check computed from uncompressed data (default : enabled) +add integrity check computed from uncompressed data (default : enabled) . .TP \fB\-\-\fR @@ -226,6 +230,9 @@ set process priority to real\-time . .SH "ADVANCED COMPRESSION OPTIONS" . +.SS "\-B#:" +Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\. +. .SS "\-\-zstd[=options]:" \fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR: . @@ -293,6 +300,13 @@ A larger minimum match length usually improves compression ratio but decreases c .IP The minimum \fItlen\fR is 4 and the maximum is 999\. . +.TP +\fBoverlapLog\fR=\fIovlog\fR, \fBovlog\fR=\fIovlog\fR +Select the amount of data reloaded from previous job into next one\. Reloading more data improves compression ratio, but decreases speed\. This parameter is only available if multithreading is enabled\. +. +.IP +The minimum \fIovlog\fR is 0, and the maximum is 9\. 0 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the amount of reload by a factor 2\. Default \fIovlog\fR is 6, which means "reload \fBwindowSize / 8\fR"\. Exception : the maximum compression level (22) has a default \fIovlog\fR of 9\. +. .SS "Example" The following parameters sets advanced compression options to those of predefined level 19 for files bigger than 256 KB: . diff --git a/programs/zstd.1.md b/programs/zstd.1.md index d1161a522..f8ef513ea 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -99,6 +99,9 @@ the last one takes effect. * `--ultra`: unlocks high compression levels 20+ (maximum 22), using a lot more memory. Note that decompression will also require more memory when using these levels. +* `-T#`: + Compress using # threads (default: 1). + This modifier is only available if `zstd` was compiled with multithreading support. * `-D file`: use `file` as Dictionary to compress or decompress FILE(s) * `--nodictID`: @@ -123,7 +126,7 @@ the last one takes effect. remove source file(s) after successful compression or decompression * `-k`, `--keep`: keep source file(s) after successful compression or decompression. - This is the default behaviour. + This is the default behavior. * `-r`: operate recursively on dictionaries * `-h`/`-H`, `--help`: @@ -136,10 +139,11 @@ the last one takes effect. suppress warnings, interactivity, and notifications. specify twice to suppress errors too. * `-C`, `--[no-]check`: - add integrety check computed from uncompressed data (default : enabled) + add integrity check computed from uncompressed data (default : enabled) * `--`: All arguments after `--` are treated as files + DICTIONARY BUILDER ------------------ `zstd` offers _dictionary_ compression, @@ -218,8 +222,17 @@ BENCHMARK * `--priority=rt`: set process priority to real-time + ADVANCED COMPRESSION OPTIONS ---------------------------- +### -B#: +Select the size of each compression job. +This parameter is available only when multi-threading is enabled. +Default value is `4 * windowSize`, which means it varies depending on compression level. +`-B#` makes it possible to select a custom value. +Note that job size must respect a minimum value which is enforced transparently. +This minimum is either 1 MB, or `overlapSize`, whichever is largest. + ### --zstd[=options]: `zstd` provides 22 predefined compression levels. The selected or default predefined compression level can be changed with @@ -290,6 +303,19 @@ The list of available _options_: The minimum _tlen_ is 4 and the maximum is 999. +- `overlapLog`=_ovlog_, `ovlog`=_ovlog_: + Select the amount of data reloaded from previous job into next one. + Reloading more data improves compression ratio, but decreases speed. + This parameter is only available if multithreading is enabled. + + The minimum _ovlog_ is 0, and the maximum is 9. + 0 means "no overlap", hence completely independent jobs. + 9 means "full overlap", meaning up to `windowSize` is reloaded from previous job. + Reducing _ovlog_ by 1 reduces the amount of reload by a factor 2. + Default _ovlog_ is 6, which means "reload `windowSize / 8`". + Exception : the maximum compression level (22) has a default _ovlog_ of 9. + + ### Example The following parameters sets advanced compression options to those of predefined level 19 for files bigger than 256 KB: diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 0cf8cacd1..18e259c74 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -113,19 +113,20 @@ static int usage_advanced(const char* programName) DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL); + DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n"); DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); -#ifdef UTIL_HAS_CREATEFILELIST - DISPLAY( " -r : operate recursively on directories \n"); -#endif #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); - DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); - DISPLAY( "--[no-]check : integrity check (default:enabled) \n"); #ifdef ZSTD_MULTITHREAD DISPLAY( " -T# : use # threads for compression (default:1) \n"); - DISPLAY( " -B# : select size of independent sections (default:0==automatic) \n"); + DISPLAY( " -B# : select size of each job (default:0==automatic) \n"); +#endif + DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); + DISPLAY( "--[no-]check : integrity check (default:enabled) \n"); +#endif +#ifdef UTIL_HAS_CREATEFILELIST + DISPLAY( " -r : operate recursively on directories \n"); #endif #ifdef ZSTD_GZCOMPRESS DISPLAY( "--format=gzip : compress files to the .gz format \n"); @@ -134,7 +135,6 @@ static int usage_advanced(const char* programName) DISPLAY( "--format=xz : compress files to the .xz format \n"); DISPLAY( "--format=lzma : compress files to the .lzma format \n"); #endif -#endif #ifndef ZSTD_NODECOMPRESS DISPLAY( "--test : test compressed file integrity \n"); #if ZSTD_SPARSE_DEFAULT