From 64bc441d7d06b267aba52cfdae681a5582d4cfe7 Mon Sep 17 00:00:00 2001
From: Sen Huang <senhuang96@fb.com>
Date: Thu, 3 Oct 2019 13:53:04 -0400
Subject: [PATCH] Now constructs final destination path without allocating new
 table

---
 programs/fileio.c  | 184 +++++++++++++++++++++++++++++++++++----------
 programs/fileio.h  |  14 ++--
 programs/util.c    |  93 -----------------------
 programs/util.h    |   5 --
 programs/zstdcli.c |  37 +--------
 5 files changed, 157 insertions(+), 176 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 6006af01b..9988e2d6a 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -628,6 +628,104 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
     return (size_t)fileSize;
 }
 
+
+
+/* FIO_checkFilenameCollisions() :
+ * Checks for and warns if there are any files that would have the same output path
+ */
+int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
+    const char** filenameTableSorted;
+    const char* c, *prevElem;
+    char* filename;
+    unsigned u;
+
+    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
+    c = "\\";
+    #else
+    c = "/";
+    #endif
+
+    filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
+    if (!filenameTableSorted) {
+        DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
+        return 1;
+    }
+    
+    for (u = 0; u < nbFiles; ++u) {
+        filename = strrchr(filenameTable[u], c[0]);
+        if (filename == NULL) {
+            filenameTableSorted[u] = filenameTable[u];
+        } else {
+            filenameTableSorted[u] = filename+1;
+        }
+    }
+
+    qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
+    prevElem = filenameTableSorted[0];
+    for (u = 1; u < nbFiles; ++u) {
+        if (strcmp(prevElem, filenameTableSorted[u]) == 0) {
+            DISPLAY("WARNING: Two files have same filename: %s\n", prevElem);
+        }
+        prevElem = filenameTableSorted[u];
+    }
+
+    free(filenameTableSorted);
+    return 0;
+}
+
+/* FIO_determineDstFilenameOutdir() :
+ * Takes a source file name and specified output directory, and
+ * allocates memory for and returns a pointer to final path.
+ * This function never returns an error (it may abort() in case of pb)
+ */
+static char*
+FIO_determineDstFilenameOutdir(const char* srcFilename, const char* outDirName, const size_t suffixLen)
+{
+    const char* c, *filenameBegin;
+    char* filename, *result;
+    size_t finalPathLen;
+
+    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
+    c = "\\";
+    #else
+    c = "/";
+    #endif
+
+    finalPathLen = strlen(outDirName);
+    filenameBegin = strrchr(srcFilename, c[0]);
+    if (filenameBegin == NULL) {
+        filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char));
+        if (!filename) {
+            EXM_THROW(30, "zstd: %s", strerror(errno));
+        }
+        strcpy(filename, srcFilename);
+    } else {
+        filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
+        if (!filename) {
+            EXM_THROW(30, "zstd: %s", strerror(errno));
+        }
+        strcpy(filename, filenameBegin+1);
+    }
+
+    finalPathLen += strlen(filename);
+    result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char));
+    if (!result) {
+        free(filename);
+        EXM_THROW(30, "zstd: %s", strerror(errno));
+    }
+
+    strcpy(result, outDirName);
+    if (outDirName[strlen(outDirName)-1] == c[0]) {
+        strcat(result, filename);
+    } else {  
+        strcat(result, c);
+        strcat(result, filename);
+    }
+
+    free(filename);
+    return result;
+}
+
 #ifndef ZSTD_NOCOMPRESS
 
 /* **********************************************************************
@@ -1379,19 +1477,25 @@ int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName,
     return result;
 }
 
-
 /* FIO_determineCompressedName() :
  * create a destination filename for compressed srcFileName.
  * @return a pointer to it.
  * This function never returns an error (it may abort() in case of pb)
  */
 static const char*
-FIO_determineCompressedName(const char* srcFileName, const char* suffix)
+FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix)
 {
     static size_t dfnbCapacity = 0;
     static char* dstFileNameBuffer = NULL;   /* using static allocation : this function cannot be multi-threaded */
-    size_t const sfnSize = strlen(srcFileName);
+    char* outDirFilename = NULL;
+    size_t sfnSize = strlen(srcFileName);
     size_t const suffixSize = strlen(suffix);
+    if (outDirName) {
+        outDirFilename = FIO_determineDstFilenameOutdir(srcFileName, outDirName, suffixSize);
+        sfnSize = strlen(outDirFilename);
+        assert(outDirFilename != NULL);
+    }
+    
     if (dfnbCapacity <= sfnSize+suffixSize+1) {
         /* resize buffer for dstName */
         free(dstFileNameBuffer);
@@ -1399,9 +1503,16 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
         dstFileNameBuffer = (char*)malloc(dfnbCapacity);
         if (!dstFileNameBuffer) {
             EXM_THROW(30, "zstd: %s", strerror(errno));
-    }   }
+        }
+    }
     assert(dstFileNameBuffer != NULL);
-    memcpy(dstFileNameBuffer, srcFileName, sfnSize);
+
+    if (outDirFilename) {
+        memcpy(dstFileNameBuffer, outDirFilename, sfnSize);
+        free(outDirFilename);
+    } else {
+        memcpy(dstFileNameBuffer, srcFileName, sfnSize);
+    }
     memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
     return dstFileNameBuffer;
 }
@@ -1414,24 +1525,17 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
  * or into a destination folder (specified with -O)
  */
 int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
-                                  const char* outDirName, char** dstFileNamesTable, 
-                                  unsigned nbFiles, const char* outFileName,
-                                  const char* suffix, const char* dictFileName,
-                                  int compressionLevel, ZSTD_compressionParameters comprParams)
+                                  const char* outDirName, unsigned nbFiles, 
+                                  const char* outFileName, const char* suffix, 
+                                  const char* dictFileName, int compressionLevel,
+                                  ZSTD_compressionParameters comprParams)
 {
     int error = 0;
     cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams);
 
     /* init */
     assert(outFileName != NULL || suffix != NULL);
-    if (outDirName != NULL) {   /* output into a particular folder */
-        unsigned u;
-        for (u = 0; u < nbFiles; ++u) {
-            const char* const srcFileName = inFileNamesTable[u];
-            const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix);
-            error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
-        }
-    } else if (outFileName != NULL) {   /* output into a single destination (stdout typically) */
+    if (outFileName != NULL) {   /* output into a single destination (stdout typically) */
         ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
         if (ress.dstFile == NULL) {  /* could not open outFileName */
             error = 1;
@@ -1448,9 +1552,11 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN
         unsigned u;
         for (u=0; u<nbFiles; u++) {
             const char* const srcFileName = inFileNamesTable[u];
-            const char* const dstFileName = FIO_determineCompressedName(srcFileName, suffix);  /* cannot fail */
+            const char* const dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix);  /* cannot fail */
             error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
-    }   }
+        }
+        FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles);
+    }
 
     FIO_freeCResources(ress);
     return error;
@@ -2169,13 +2275,14 @@ int FIO_decompressFilename(FIO_prefs_t* const prefs,
  * @return a pointer to it.
  * @return == NULL if there is an error */
 static const char*
-FIO_determineDstName(const char* srcFileName)
+FIO_determineDstName(const char* srcFileName, const char* outDirName)
 {
     static size_t dfnbCapacity = 0;
     static char* dstFileNameBuffer = NULL;   /* using static allocation : this function cannot be multi-threaded */
-
-    size_t const sfnSize = strlen(srcFileName);
+    char* dstFilenameOutDir = NULL;
+    size_t sfnSize = strlen(srcFileName);
     size_t suffixSize;
+    
     const char* const suffixPtr = strrchr(srcFileName, '.');
     if (suffixPtr == NULL) {
         DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
@@ -2213,19 +2320,29 @@ FIO_determineDstName(const char* srcFileName)
                      srcFileName, suffixlist);
         return NULL;
     }
+    if (outDirName) {
+        dstFilenameOutDir = FIO_determineDstFilenameOutdir(srcFileName, outDirName, 0);
+        sfnSize = strlen(dstFilenameOutDir);
+        assert(dstFilenameOutDir != NULL);
+    }
 
-    /* allocate enough space to write dstFilename into it */
     if (dfnbCapacity+suffixSize <= sfnSize+1) {
+        /* allocate enough space to write dstFilename into it */
         free(dstFileNameBuffer);
         dfnbCapacity = sfnSize + 20;
         dstFileNameBuffer = (char*)malloc(dfnbCapacity);
         if (dstFileNameBuffer==NULL)
-            EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
+            EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno)); 
     }
 
     /* return dst name == src name truncated from suffix */
     assert(dstFileNameBuffer != NULL);
-    memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
+    if (dstFilenameOutDir) {
+        memcpy(dstFileNameBuffer, dstFilenameOutDir, sfnSize - suffixSize);
+        free(dstFilenameOutDir);
+    } else {
+        memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
+    }
     dstFileNameBuffer[sfnSize-suffixSize] = '\0';
     return dstFileNameBuffer;
 
@@ -2236,23 +2353,13 @@ FIO_determineDstName(const char* srcFileName)
 int
 FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
                                 const char** srcNamesTable, unsigned nbFiles,
-                                const char* outDirName, char** dstFileNamesTable, 
-                                const char* outFileName,
+                                const char* outDirName, const char* outFileName,
                                 const char* dictFileName)
 {
     int error = 0;
     dRess_t ress = FIO_createDResources(prefs, dictFileName);
 
-    if (outDirName != NULL) {   /* output into a particular folder */
-        unsigned u;
-        for (u = 0; u < nbFiles; ++u) {
-            const char* const srcFileName = srcNamesTable[u];
-            const char* const dstFileName = FIO_determineDstName(dstFileNamesTable[u]);
-            if (dstFileName == NULL) { error=1; continue; }
-            
-            error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
-        }
-    } else if (outFileName) {
+    if (outFileName) {
         unsigned u;
         ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
         if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
@@ -2265,11 +2372,12 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
         unsigned u;
         for (u=0; u<nbFiles; u++) {   /* create dstFileName */
             const char* const srcFileName = srcNamesTable[u];
-            const char* const dstFileName = FIO_determineDstName(srcFileName);
+            const char* const dstFileName = FIO_determineDstName(srcFileName, outDirName);
             if (dstFileName == NULL) { error=1; continue; }
 
             error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
         }
+        FIO_checkFilenameCollisions(srcNamesTable ,nbFiles);
     }
 
     FIO_freeDResources(ress);
diff --git a/programs/fileio.h b/programs/fileio.h
index 40cd8985f..ded1c0373 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -105,19 +105,23 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
 /** FIO_compressMultipleFilenames() :
     @return : nb of missing files */
 int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
-                                  const char* outDirName, char** dstFileNamesTable, 
-                                  unsigned nbFiles, const char* outFileName,
-                                  const char* suffix, const char* dictFileName,
-                                  int compressionLevel, ZSTD_compressionParameters comprParams);
+                                  const char* outDirName, unsigned nbFiles, 
+                                  const char* outFileName, const char* suffix, 
+                                  const char* dictFileName, int compressionLevel, 
+                                  ZSTD_compressionParameters comprParams);
 
 /** FIO_decompressMultipleFilenames() :
     @return : nb of missing or skipped files */
 int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
                                     const char** srcNamesTable, unsigned nbFiles,
-                                    const char* outDirName, char** dstFilenameTable, 
+                                    const char* outDirName,
                                     const char* outFileName,
                                     const char* dictFileName);
 
+/* FIO_checkFilenameCollisions() :
+ * Checks for and warns if thereå are any files that would have the same output path
+ */
+int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
 
 /*-*************************************
 *  Advanced stuff (should actually be hosted elsewhere)
diff --git a/programs/util.c b/programs/util.c
index c12a31887..2e5815968 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -94,99 +94,6 @@ int UTIL_compareStr(const void *p1, const void *p2) {
     return strcmp(* (char * const *) p1, * (char * const *) p2);
 }
 
-int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles) {
-    char** dstFilenameTableSorted;
-    char* prevElem;
-    unsigned u;
-
-    dstFilenameTableSorted = (char**) malloc(sizeof(char*) * nbFiles);
-    if (!dstFilenameTableSorted) {
-        UTIL_DISPLAYLEVEL(1, "Unable to malloc new str array, not checking for name collisions\n");
-        return 1;
-    }
-
-    for (u = 0; u < nbFiles; ++u) {
-        dstFilenameTableSorted[u] = dstFilenameTable[u];
-    }
-    qsort(dstFilenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr);
-    prevElem = dstFilenameTableSorted[0];
-    for (u = 1; u < nbFiles; ++u) {
-        if (strcmp(prevElem, dstFilenameTableSorted[u]) == 0) {
-            UTIL_DISPLAYLEVEL(1, "WARNING: Two files have same filename as source : %s\n", prevElem);
-        }
-        prevElem = dstFilenameTableSorted[u];
-    }
-
-    free(dstFilenameTableSorted);
-    return 0;
-}
-
-void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable,
-        const unsigned nbFiles, const char* outDirName)
-{
-    unsigned u;
-    const char* c;
-    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
-    c = "\\";
-    #else
-    c = "/";
-    #endif
-
-    for (u = 0; u < nbFiles; ++u) {
-        char* filename;
-        const char* filenameBegin;
-        size_t finalPathLen;
-        finalPathLen = strlen(outDirName);
-        filenameBegin = strrchr(filenameTable[u], c[0]);
-        if (filenameBegin == NULL) {
-            filename = (char*) malloc((strlen(filenameTable[u])+1) * sizeof(char));
-            if (!filename) {
-                UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n");
-                continue;
-            }
-            strcpy(filename, filenameTable[u]);
-        } else {
-            filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
-            if (!filename) {
-                UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n");
-                continue;
-            }
-            strcpy(filename, filenameBegin+1);
-        }
-
-        finalPathLen += strlen(filename);
-        dstFilenameTable[u] = (char*) malloc((finalPathLen+3) * sizeof(char));
-        if (!dstFilenameTable[u]) {
-            UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n");
-            free(filename);
-            continue;
-        }
-
-        strcpy(dstFilenameTable[u], outDirName);
-        if (outDirName[strlen(outDirName)-1] == c[0]) {
-            strcat(dstFilenameTable[u], filename);
-        } else {  
-            strcat(dstFilenameTable[u], c);
-            strcat(dstFilenameTable[u], filename);
-        }
-
-        free(filename);
-    }
-
-    if (UTIL_checkFilenameCollisions(dstFilenameTable, nbFiles)) {
-        UTIL_DISPLAYLEVEL(1, "Checking for filename collisions failed");
-    }
-}
-
-void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) {
-    unsigned u;
-    for (u = 0; u < nbFiles; ++u) {
-        if (dstDirTable[u] != NULL)
-            free(dstDirTable[u]);
-    }
-    if (dstDirTable != NULL) free((void*)dstDirTable);
-}
-
 int UTIL_isSameFile(const char* file1, const char* file2)
 {
 #if defined(_MSC_VER)
diff --git a/programs/util.h b/programs/util.h
index 558e7a9c6..97d641c18 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -130,11 +130,6 @@ U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 int UTIL_compareStr(const void *p1, const void *p2);
-int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles);
-/* Populates dstFilenameTable using outDirName concatenated with entries from filenameTable */
-void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, const unsigned nbFiles,
-    const char* outDirName);
-void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles);
 
 U32 UTIL_isLink(const char* infilename);
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 91e67a2d3..ae53d2c39 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -585,7 +585,6 @@ int main(int argCount, const char* argv[])
     unsigned recursive = 0;
     unsigned memLimit = 0;
     const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));   /* argCount >= 1 */
-    char** dstFilenameTable;
     unsigned filenameIdx = 0;
     const char* programName = argv[0];
     const char* outFileName = NULL;
@@ -1175,26 +1174,10 @@ int main(int argCount, const char* argv[])
         if (adaptMin > cLevel) cLevel = adaptMin;
         if (adaptMax < cLevel) cLevel = adaptMax;
 
-        if (outDirName) {
-            if (UTIL_isDirectory(outDirName)) {
-                DISPLAY("Output of files will be in directory: %s\n", outDirName);
-                dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
-                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName);
-            } else {
-                DISPLAY("%s is not a directory!\n", outDirName);
-                CLEAN_RETURN(1);
-            }
-        } else {
-            dstFilenameTable = NULL;
-        }
-
         if ((filenameIdx==1) && outFileName)
           operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
         else
-          operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
-        
-        if (dstFilenameTable)
-            UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx);
+          operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
         (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
         DISPLAY("Compression not supported \n");
@@ -1209,26 +1192,10 @@ int main(int argCount, const char* argv[])
             }
         }
         FIO_setMemLimit(prefs, memLimit);
-
-        if (outDirName) {
-            if (UTIL_isDirectory(outDirName)) {
-                DISPLAY("Output of files will be in directory: %s\n", outDirName);
-                dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*));
-                UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName);
-            } else {
-                DISPLAY("%s is not a directory!\n", outDirName);
-                CLEAN_RETURN(1);
-            }
-        } else {
-            dstFilenameTable = NULL;
-        }
-        
         if (filenameIdx==1 && outFileName)
             operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
         else
-            operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName);
-        if (dstFilenameTable)
-            UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx);
+            operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName);
 #else
         DISPLAY("Decompression not supported \n");
 #endif