updated man page

NEWS

@@ -5,6 +5,7 @@ Improved : slightly better compression ratio at --ultra levels (>= 20)
 Added : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
 Added : example/multiple_streaming_compression
 Changed : zstd_errors.h is now part of include installation
+Updated man page
 Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
 
 v1.1.0

examples/simple_decompression.c

@@ -65,7 +65,7 @@ static void decompress(const char* fname)
     void* const cBuff = loadFile_X(fname, &cSize);
     unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
     if (rSize==0) {
-        printf("%s : original size unknown \n", fname);
+        printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
         exit(5);
     }
     void* const rBuff = malloc_X((size_t)rSize);

examples/streaming_decompression.c

@@ -71,9 +71,12 @@ static void decompressFile_orDie(const char* fname)
 
     ZSTD_DStream* const dstream = ZSTD_createDStream();
     if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
+
+    /* In more complex scenarios, a file may consist of multiple appended frames (ex : pzstd).
+    *  The following example decompresses only the first frame.
+    *  It is compatible with other provided streaming examples */
     size_t const initResult = ZSTD_initDStream(dstream);
     if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
-
     size_t read, toRead = initResult;
     while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
         ZSTD_inBuffer input = { buffIn, read, 0 };

programs/zstd.1

@@ -35,22 +35,85 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
 \fBzstd\fR command line is generally similar to gzip, but features the following differences :
  - Source files are preserved by default
    It's possible to remove them automatically by using \fB--rm\fR command
- - By default, when compressing a single file, \fBzstd\fR displays progress notifications and result summary.
+ - When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default.
      Use \fB-q\fR to turn them off
 
 
 
 .SH OPTIONS
+
+.
+.SS "Integer suffixes and special values"
+In most places where an integer argument is expected,
+an optional suffix is supported to easily indicate large integers.
+There must be no space between the integer and the suffix.
+.TP
+.B KiB
+Multiply the integer by 1,024 (2^10).
+.BR Ki ,
+.BR K ,
+and
+.B KB
+are accepted as synonyms for
+.BR KiB .
+.TP
+.B MiB
+Multiply the integer by 1,048,576 (2^20).
+.BR Mi ,
+.BR M ,
+and
+.B MB
+are accepted as synonyms for
+.BR MiB .
+
+.
+.SS "Operation mode"
+If multiple operation mode options are given,
+the last one takes effect.
+.TP
+.BR \-z ", " \-\-compress
+Compress.
+This is the default operation mode when no operation mode option
+is specified and no other operation mode is implied from
+the command name (for example,
+.B unzstd
+implies
+.BR \-\-decompress ).
+.TP
+.BR \-d ", " \-\-decompress ", " \-\-uncompress
+Decompress.
+.TP
+.BR \-t ", " \-\-test
+Test the integrity of compressed
+.IR files .
+This option is equivalent to
+.B "\-\-decompress \-\-stdout"
+except that the decompressed data is discarded instead of being
+written to standard output.
+No files are created or removed.
+.TP
+.B \-b#
+ benchmark file(s) using compression level #
+.TP
+.B \--train FILEs
+ use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100).
+
+.
+.SS "Operation modifiers"
 .TP
 .B \-#
- # compression level [1-22] (default:3)
+ # compression level [1-19] (default:3)
 .TP
-.BR \-d ", " --decompress
+.BR \--ultra
- decompression
+ unlocks high compression levels 20+ (maximum 22), using a lot more memory
 .TP
 .B \-D file
  use `file` as Dictionary to compress or decompress FILE(s)
 .TP
+.BR \--no-dictID
+ do not store dictionary ID within frame header (dictionary compression).
+ the decoder will have to rely on implicit knowledge about which dictionary to use, it won't be able to check if it's correct.
+.TP
 .B \-o file
  save result into `file` (only possible with a single INPUT-FILE)
 .TP
@@ -60,6 +123,11 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
 .BR \-c ", " --stdout
  force write to standard output, even if it is the console
 .TP
+.BR \--[no-]sparse
+ enable / disable sparse FS support, to make files with many zeroes smaller on disk.
+ default : enabled when output is into a file, and disabled when output is stdout.
+ This setting overrides default and can force sparse mode over stdout.
+.TP
 .BR \--rm
  remove source file(s) after successful compression or decompression
 .TP
@@ -83,8 +151,8 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
  suppress warnings, interactivity and notifications.
  specify twice to suppress errors too.
 .TP
-.BR \-C ", " --check
+.BR \-C ", " --[no-]check
- add integrity check computed from uncompressed data
+ add integrity check computed from uncompressed data (default : enabled)
 .TP
 .BR \-t ", " --test
  Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
@@ -103,10 +171,8 @@ It will improve compression ratio of small files.
 Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
 .TP
 .B \--train FILEs
- use FILEs as training set to create a dictionary.
+ use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100),
- The training set should contain a lot of small files (> 100).
+and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)
- and weight typically 100x the target dictionary size
- (for example, 10 MB for a 100 KB dictionary)
 .TP
 .B \-o file
  dictionary saved into `file` (default: dictionary)
@@ -131,6 +197,9 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
 .B \-b#
  benchmark file(s) using compression level #
 .TP
+.B \-e#
+ benchmark file(s) using multiple compression levels, from -b# to -e# (included).
+.TP
 .B \-i#
  minimum evaluation time, in seconds (default : 3s), benchmark mode only
 .TP

programs/zstdcli.c

@@ -199,26 +199,24 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
     return result;
 }
 
+typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode;
 
 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
 
 int main(int argCount, const char* argv[])
 {
     int argNb,
-        bench=0,
-        decode=0,
-        testmode=0,
         forceStdout=0,
         main_pause=0,
         nextEntryIsDictionary=0,
         operationResult=0,
-        dictBuild=0,
         nextArgumentIsOutFileName=0,
         nextArgumentIsMaxDict=0,
         nextArgumentIsDictID=0,
         nextArgumentIsFile=0,
         ultra=0,
         lastCommand = 0;
+    zstd_operation_mode operation = zom_compress;
     int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
     int cLevelLast = 1;
     unsigned recursive = 0;
@@ -241,7 +239,7 @@ int main(int argCount, const char* argv[])
     /* init */
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
     (void)dictCLevel; (void)dictSelect; (void)dictID;  /* not used when ZSTD_NODICT set */
-    (void)decode; (void)cLevel; (void)testmode; /* not used when ZSTD_NOCOMPRESS set */
+    (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
     (void)ultra; (void)memLimit;   /* not used when ZSTD_NODECOMPRESS set */
     if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
     filenameTable[0] = stdinmark;
@@ -253,8 +251,8 @@ int main(int argCount, const char* argv[])
     }
 
     /* preset behaviors */
-    if (!strcmp(programName, ZSTD_UNZSTD)) decode=1;
+    if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress;
-    if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
+    if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
 
     /* command switches */
     for (argNb=1; argNb<argCount; argNb++) {
@@ -265,7 +263,9 @@ int main(int argCount, const char* argv[])
 
             /* long commands (--long-word) */
             if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; }   /* only file names allowed from now on */
-            if (!strcmp(argument, "--decompress")) { decode=1; continue; }
+            if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
+            if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
+            if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
             if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; }
             if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
             if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
@@ -278,8 +278,8 @@ int main(int argCount, const char* argv[])
             if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
             if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
             if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
-            if (!strcmp(argument, "--test")) { testmode=1; decode=1; continue; }
+            if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
-            if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
+            if (!strcmp(argument, "--train")) { operation=zom_train; outFileName=g_defaultDictName; continue; }
             if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; }
             if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; }
             if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
@@ -323,8 +323,11 @@ int main(int argCount, const char* argv[])
                     case 'H':
                     case 'h': displayOut=stdout; CLEAN_RETURN(usage_advanced(programName));
 
+                         /* Compress */
+                    case 'z': operation=zom_compress; argument++; break;
+
                          /* Decoding */
-                    case 'd': decode=1; argument++; break;
+                    case 'd': operation=zom_decompress; argument++; break;
 
                         /* Force stdout, even if stdout==console */
                     case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
@@ -348,7 +351,7 @@ int main(int argCount, const char* argv[])
                     case 'C': argument++; FIO_setChecksumFlag(2); break;
 
                         /* test compressed file */
-                    case 't': testmode=1; decode=1; argument++; break;
+                    case 't': operation=zom_test; argument++; break;
 
                         /* destination file name */
                     case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
@@ -366,7 +369,7 @@ int main(int argCount, const char* argv[])
 
 #ifndef ZSTD_NOBENCH
                         /* Benchmark */
-                    case 'b': bench=1; argument++; break;
+                    case 'b': operation=zom_train; argument++; break;
 
                         /* range bench (benchmark only) */
                     case 'e':
@@ -470,7 +473,7 @@ int main(int argCount, const char* argv[])
 #endif
 
     /* Check if benchmark is selected */
-    if (bench) {
+    if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH
         BMK_setNotificationLevel(displayLevel);
         BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
@@ -479,7 +482,7 @@ int main(int argCount, const char* argv[])
     }
 
     /* Check if dictionary builder is selected */
-    if (dictBuild) {
+    if (operation==zom_train) {
 #ifndef ZSTD_NODICT
         ZDICT_params_t dictParams;
         memset(&dictParams, 0, sizeof(dictParams));
@@ -498,7 +501,7 @@ int main(int argCount, const char* argv[])
 
     /* Check if input/output defined as console; trigger an error in this case */
     if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));
-    if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && strcmp(filenameTable[0], stdinmark) && !(forceStdout && decode))
+    if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && strcmp(filenameTable[0], stdinmark) && !(forceStdout && (operation==zom_decompress)))
         CLEAN_RETURN(badusage(programName));
 
     /* user-selected output filename, only possible with a single file */
@@ -522,7 +525,7 @@ int main(int argCount, const char* argv[])
 
     /* IO Stream/File */
     FIO_setNotificationLevel(displayLevel);
-    if (!decode) {
+    if (operation==zom_compress) {
 #ifndef ZSTD_NOCOMPRESS
         if ((filenameIdx==1) && outFileName)
           operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
@@ -531,9 +534,9 @@ int main(int argCount, const char* argv[])
 #else
         DISPLAY("Compression not supported\n");
 #endif
-    } else {  /* decompression */
+    } else {  /* decompression or test */
 #ifndef ZSTD_NODECOMPRESS
-        if (testmode) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
+        if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
         FIO_setMemLimit(memLimit);
         if (filenameIdx==1 && outFileName)
             operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);