cli : added ability to compress multiple files into stdout (-c)

lib/zstd_buffered.h

@@ -1,6 +1,6 @@
 /*
     Buffered version of Zstd compression library
-    Copyright (C) 2015, Yann Collet.
+    Copyright (C) 2015-2016, Yann Collet.
 
     BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
@@ -26,14 +26,13 @@
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
+    - zstd homepage : http://www.zstd.net/
-    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 #ifndef ZSTD_BUFFERED_H
 #define ZSTD_BUFFERED_H
 
 /* The objects defined into this file should be considered experimental.
- * They are not labelled stable, as their prototype may change in the future.
+ * They are not considered stable, as their prototype may change in the future.
  * You can use them for tests, provide feedback, or if you can endure risk of future changes.
  */
 
@@ -42,7 +41,7 @@ extern "C" {
 #endif
 
 /* *************************************
-*  Includes
+*  Dependencies
 ***************************************/
 #include <stddef.h>   /* size_t */
 
@@ -75,7 +74,7 @@ ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* d
 ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
 ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
 
-/** ************************************************
+/*-*************************************************
 *  Streaming compression
 *
 *  A ZBUFF_CCtx object is required to track streaming operation.
@@ -123,12 +122,14 @@ ZSTDLIB_API size_t      ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
 ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
 ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
 
-ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
+ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
+                                            void* dst, size_t* dstCapacityPtr,
+                                      const void* src, size_t* srcSizePtr);
 
-/** ************************************************
+/*-***************************************************************************
 *  Streaming decompression
 *
-*  A ZBUFF_DCtx object is required to track streaming operation.
+*  A ZBUFF_DCtx object is required to track streaming operations.
 *  Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
 *  Use ZBUFF_decompressInit() to start a new decompression operation,
 *   or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
@@ -143,10 +144,10 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t*
 *            or 0 when a frame is completely decoded
 *            or an error code, which can be tested using ZBUFF_isError().
 *
-*  Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize
+*  Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() / ZBUFF_recommendedDOutSize()
-*  output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
+*  output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
 *  input  : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
-* **************************************************/
+* *******************************************************************************/
 
 
 /* *************************************
@@ -155,7 +156,7 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t*
 ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
 ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
 
-/** The below functions provide recommended buffer sizes for Compression or Decompression operations.
+/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
 *   These sizes are just hints, and tend to offer better latency */
 ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
 ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);

lib/zstd_buffered_static.h

@@ -1,7 +1,7 @@
 /*
     zstd - buffered version of compression library
     experimental complementary API, for static linking only
-    Copyright (C) 2015, Yann Collet.
+    Copyright (C) 2015-2016, Yann Collet.
 
     BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
@@ -27,8 +27,7 @@
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
     You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
+    - zstd homepage : http://www.zstd.net
-    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 #ifndef ZSTD_BUFFERED_STATIC_H
 #define ZSTD_BUFFERED_STATIC_H

programs/fileio.c

@@ -186,54 +186,73 @@ static U64 FIO_getFileSize(const char* infilename)
 }
 
 
+static FILE* FIO_openSrcFile(const char* srcFileName)
+{
+    FILE* f;
+
+    if (!strcmp (srcFileName, stdinmark)) {
+        DISPLAYLEVEL(4,"Using stdin for input\n");
+        f = stdin;
+        SET_BINARY_MODE(stdin);
+    } else {
+        f = fopen(srcFileName, "rb");
+    }
+
+    if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: No such file\n", srcFileName);
+
+    return f;
+}
+
+
+static FILE* FIO_openDstFile(const char* dstFileName)
+{
+    FILE* f;
+
+    if (!strcmp (dstFileName, stdoutmark)) {
+        DISPLAYLEVEL(4,"Using stdout for output\n");
+        f = stdout;
+        SET_BINARY_MODE(stdout);
+    } else {
+        if (!g_overwrite) {  /* Check if destination file already exists */
+            f = fopen( dstFileName, "rb" );
+            if (f != 0) {  /* dest file exists, prompt for overwrite authorization */
+                fclose(f);
+                if (g_displayLevel <= 1) {
+                    /* No interaction possible */
+                    DISPLAY("zstd: %s already exists; not overwritten  \n", dstFileName);
+                    return 0;
+                }
+                DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
+                {
+                    int ch = getchar();
+                    if ((ch!='Y') && (ch!='y')) {
+                        DISPLAY("    not overwritten  \n");
+                        return 0;
+                    }
+                    while ((ch!=EOF) && (ch!='\n')) ch = getchar();  /* flush rest of input line */
+        }   }   }
+        f = fopen( dstFileName, "wb" );
+    }
+    return f;
+}
+
+
 static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
                         const char* dstFileName, const char* srcFileName)
 {
-    if (!strcmp (srcFileName, stdinmark)) {
+    *fileInPtr = FIO_openSrcFile(srcFileName);
-        DISPLAYLEVEL(4,"Using stdin for input\n");
-        *fileInPtr = stdin;
-        SET_BINARY_MODE(stdin);
-    } else {
-        *fileInPtr = fopen(srcFileName, "rb");
-    }
-
     if ( *fileInPtr==0 ) {
         DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName);
         return 1;
     }
 
-    if (!strcmp (dstFileName, stdoutmark)) {
+    *fileOutPtr = FIO_openDstFile(dstFileName);
-        DISPLAYLEVEL(4,"Using stdout for output\n");
+    if (*fileOutPtr==0) return 1;
-        *fileOutPtr = stdout;
-        SET_BINARY_MODE(stdout);
-    } else {
-        if (!g_overwrite) {  /* Check if destination file already exists */
-            *fileOutPtr = fopen( dstFileName, "rb" );
-            if (*fileOutPtr != 0) {  /* dest file exists, prompt for overwrite authorization */
-                fclose(*fileOutPtr);
-                DISPLAY("Warning : %s already exists \n", dstFileName);
-                if ((g_displayLevel <= 1) || (*fileInPtr == stdin)) {
-                    /* No interaction possible */
-                    DISPLAY("Operation aborted : %s already exists \n", dstFileName);
-                    return 1;
-                }
-                DISPLAY("Overwrite ? (y/N) : ");
-                {
-                    int ch = getchar();
-                    if ((ch!='Y') && (ch!='y')) {
-                        DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
-                        return 1;
-                    }
-                    while ((ch!=EOF) && (ch!='\n')) ch = getchar();  /* flush rest of input line */
-        }   }   }
-        *fileOutPtr = fopen( dstFileName, "wb" );
-    }
-
-    if (*fileOutPtr==0) EXM_THROW(13, "Pb opening %s", dstFileName);
 
     return 0;
 }
 
+
 /*!FIO_loadFile
 *  creates a buffer, pointed by *bufferPtr,
 *  loads "filename" content into it
@@ -280,6 +299,7 @@ typedef struct {
     void*  dictBuffer;
     size_t dictBufferSize;
     ZBUFF_CCtx* ctx;
+    FILE* dstFile;
 } cRess_t;
 
 static cRess_t FIO_createCResources(const char* dictFileName)
@@ -313,24 +333,25 @@ static void FIO_freeCResources(cRess_t ress)
 }
 
 
-/*!
+/*! FIO_compressFilename_internal() :
- * FIO_compressFilename_extRess() :
+ *  same as FIO_compressFilename_extRess(), with ress.desFile already opened
- * @result : 0 : compression completed correctly,
+ *  @return : 0 : compression completed correctly,
  *            1 : missing or pb opening srcFileName
  */
-static int FIO_compressFilename_extRess(cRess_t ress,
+static int FIO_compressFilename_internal(cRess_t ress,
                                          const char* dstFileName, const char* srcFileName,
                                          int cLevel)
 {
     FILE* srcFile;
-    FILE* dstFile;
+    FILE* dstFile = ress.dstFile;
     U64 filesize = 0;
     U64 compressedfilesize = 0;
     size_t dictSize = ress.dictBufferSize;
     size_t sizeCheck, errorCode;
 
     /* File check */
-    if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
+    srcFile = FIO_openSrcFile(srcFileName);
+    if (!srcFile) return 1;   /* srcFile could not be opened */
 
     /* init */
     filesize = FIO_getFileSize(srcFileName) + dictSize;
@@ -346,8 +367,7 @@ static int FIO_compressFilename_extRess(cRess_t ress,
         filesize += inSize;
         DISPLAYUPDATE(2, "\rRead : %u MB  ", (U32)(filesize>>20));
 
-        {
+        {   /* Compress using buffered streaming */
-            /* Compress (buffered streaming ensures appropriate formatting) */
             size_t usedInSize = inSize;
             size_t cSize = ress.dstBufferSize;
             size_t result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
@@ -362,7 +382,6 @@ static int FIO_compressFilename_extRess(cRess_t ress,
             if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
             compressedfilesize += cSize;
         }
-
         DISPLAYUPDATE(2, "\rRead : %u MB  ==> %.2f%%   ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
     }
 
@@ -384,12 +403,31 @@ static int FIO_compressFilename_extRess(cRess_t ress,
 
     /* clean */
     fclose(srcFile);
-    if (fclose(dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
 
     return 0;
 }
 
 
+/*! FIO_compressFilename_extRess() :
+ *  @return : 0 : compression completed correctly,
+ *            1 : missing or pb opening srcFileName
+ */
+static int FIO_compressFilename_extRess(cRess_t ress,
+                                        const char* dstFileName, const char* srcFileName,
+                                        int cLevel)
+{
+    int result;
+
+    ress.dstFile = FIO_openDstFile(dstFileName);
+    if (ress.dstFile==0) return 1;
+
+    result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
+
+    if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
+    return result;
+}
+
+
 int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
                          const char* dictFileName, int compressionLevel)
 {
@@ -427,20 +465,28 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
     int missed_files = 0;
     char* dstFileName = (char*)malloc(FNSPACE);
     size_t dfnSize = FNSPACE;
-    const size_t suffixSize = strlen(suffix);
+    const size_t suffixSize = suffix ? strlen(suffix) : 0;
     cRess_t ress;
 
     /* init */
     ress = FIO_createCResources(dictFileName);
 
     /* loop on each file */
+    if (suffix) {
         for (u=0; u<nbFiles; u++) {
             size_t ifnSize = strlen(inFileNamesTable[u]);
             if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
             strcpy(dstFileName, inFileNamesTable[u]);
             strcat(dstFileName, suffix);
-
+            missed_files += FIO_compressFilename_extRess(ress, dstFileName,
-        missed_files += FIO_compressFilename_extRess(ress, dstFileName, inFileNamesTable[u], compressionLevel);
+                                                         inFileNamesTable[u], compressionLevel);
+        }
+    } else {
+        ress.dstFile = stdout;
+        for (u=0; u<nbFiles; u++)
+            missed_files += FIO_compressFilename_internal(ress, stdoutmark,
+                                                          inFileNamesTable[u], compressionLevel);
+        if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
     }
 
     /* Close & Free */

programs/fileio.h

@@ -64,13 +64,14 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con
 *  Multiple File functions
 ***************************************/
 /** FIO_compressMultipleFilenames() :
+    if `suffix == NULL`, output is stdout.
     @return : nb of missing files */
 int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
                                   const char* suffix,
                                   const char* dictFileName, int compressionLevel);
 
 /** FIO_decompressMultipleFilenames() :
-    @result : nb of missing or skipped files */
+    @return : nb of missing or skipped files */
 int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
                                     const char* suffix,
                                     const char* dictFileName);

programs/playTests.sh

@@ -22,7 +22,8 @@ roundTripTest() {
 
 [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
 
-printf "\n**** frame concatenation **** "
+
+echo "\n**** frame concatenation **** "
 
 echo "hello " > hello.tmp
 echo "world!" > world.tmp
@@ -37,7 +38,8 @@ rm ./*.tmp ./*.zstd
 
 echo frame concatenation test completed
 
-echo "**** flush write error test **** "
+
+echo "\n**** flush write error test **** "
 
 echo "echo foo | $ZSTD > /dev/full"
 echo foo | $ZSTD > /dev/full && die "write error not detected!"
@@ -45,27 +47,32 @@ echo "echo foo | $ZSTD | $ZSTD -d > /dev/full"
 echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
 
 
-echo "*** dictionary tests *** "
+echo "\n**** dictionary tests **** "
 
 ./datagen > tmpDict
 ./datagen -g1M | md5sum > tmp1
-./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dv | md5sum > tmp2
+./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
 diff -q tmp1 tmp2
 
-echo "*** multiple files tests *** "
+echo "\n**** multiple files tests **** "
 
 ./datagen -s1        > tmp1 2> /dev/null
 ./datagen -s2 -g100K > tmp2 2> /dev/null
 ./datagen -s3 -g1M   > tmp3 2> /dev/null
 $ZSTD -f tmp*
+echo "compress tmp* : "
 ls -ls tmp*
 rm tmp1 tmp2 tmp3
+echo "decompress tmp* : "
 $ZSTD -df *.zst
 ls -ls tmp*
+echo "compress tmp* into stdout : "
+$ZSTD -c tmp1 tmp2 tmp3 > tmpall
+ls -ls tmp*
 $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
 rm tmp*
 
-echo "**** zstd round-trip tests **** "
+echo "\n**** zstd round-trip tests **** "
 
 roundTripTest
 roundTripTest '' 6

programs/zstdcli.c

@@ -379,28 +379,28 @@ int main(int argCount, const char** argv)
     if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
     if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) return badusage(programName);
 
-    /* No warning message in pipe mode (stdin + stdout) or multiple mode */
+    /* user-selected output filename, only possible with a single file */
-    if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
+    if (outFileName && strcmp(outFileName,stdoutmark) && (filenameIdx>1)) {
-    if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1;
-
-    /* user-selected output filename only possible with a single file */
-    if ((outFileName) && (filenameIdx>1)) {
         DISPLAY("Too many files (%u) on the command line. \n", filenameIdx);
         return filenameIdx;
     }
 
+    /* No warning message in pipe mode (stdin + stdout) or multiple mode */
+    if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
+    if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1;
+
     /* IO Stream/File */
     FIO_setNotificationLevel(displayLevel);
     if (decode) {
-      if (outFileName)
+      if (filenameIdx==1)
         operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
       else
         operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName);
     } else {  /* compression */
-        if (outFileName)
+        if (filenameIdx==1)
           operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
         else
-          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName, cLevel);
+          operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, forceStdout ? NULL : ZSTD_EXTENSION, dictFileName, cLevel);
     }
 
 _end: