From 7f98b46876c475d8a0ef4f895b83730d55a60f9b Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 5 Sep 2019 16:03:35 -0700 Subject: [PATCH 01/17] adding support for -O flag: multiple files into one directory for compressions (decompression to come) --- programs/fileio.c | 37 +++++++++++++++++++++---------------- programs/fileio.h | 15 ++++++++------- programs/util.c | 43 +++++++++++++++++++++++++++++++++++++++++++ programs/util.h | 4 ++++ programs/zstdcli.c | 26 +++++++++++++++++++++++++- 5 files changed, 101 insertions(+), 24 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 20e2ee2a1..f25771864 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1276,9 +1276,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, int result; stat_t statbuf; int transfer_permissions = 0; - assert(ress.srcFile != NULL); - if (ress.dstFile == NULL) { closeDstFile = 1; DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); @@ -1369,11 +1367,9 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, return result; } - -int FIO_compressFilename(FIO_prefs_t* const prefs, - const char* dstFileName, const char* srcFileName, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams) +int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName, + const char* srcFileName, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams) { cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -1416,22 +1412,30 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) /* FIO_compressMultipleFilenames() : * compress nbFiles files - * into one destination (outFileName) - * or into one file each (outFileName == NULL, but suffix != NULL). + * into either one destination (outFileName), + * or into one file each (outFileName == NULL, but suffix != NULL), + * or into a destination folder (specified with -O) */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** inFileNamesTable, unsigned nbFiles, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams) +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, + const char* outDirName, char** dstFileNamesTable, + unsigned nbFiles, const char* outFileName, + const char* suffix, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams) { + printf("compressing multiple...\n"); int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); /* init */ assert(outFileName != NULL || suffix != NULL); - - if (outFileName != NULL) { /* output into a single destination (stdout typically) */ + if (outDirName != NULL) { /* output into a particular folder */ + unsigned u; + for (u = 0; u < nbFiles; ++u) { + const char* const srcFileName = inFileNamesTable[u]; + const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix); + error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + } + } else if (outFileName != NULL) { /* output into a single destination (stdout typically) */ ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; @@ -1453,6 +1457,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } } FIO_freeCResources(ress); + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } diff --git a/programs/fileio.h b/programs/fileio.h index 096d90b5c..1ed6d1538 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -87,8 +87,9 @@ void FIO_setNotificationLevel(int level); /** FIO_compressFilename() : @return : 0 == ok; 1 == pb with src file. */ int FIO_compressFilename (FIO_prefs_t* const prefs, - const char* outfilename, const char* infilename, const char* dictFileName, - int compressionLevel, ZSTD_compressionParameters comprParams); + const char* outfilename, const char* infilename, + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters comprParams); /** FIO_decompressFilename() : @return : 0 == ok; 1 == pb with src file. */ @@ -103,11 +104,11 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis ***************************************/ /** FIO_compressMultipleFilenames() : @return : nb of missing files */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** srcNamesTable, unsigned nbFiles, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams); +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, + const char* outDirName, char** dstFileNamesTable, + unsigned nbFiles, const char* outFileName, + const char* suffix, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams); /** FIO_decompressMultipleFilenames() : @return : nb of missing or skipped files */ diff --git a/programs/util.c b/programs/util.c index 347e76986..e8b599aaa 100644 --- a/programs/util.c +++ b/programs/util.c @@ -87,6 +87,49 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } +int UTIL_createDir(const char* outDirName) { + if (UTIL_isDirectory(outDirName)) { + return 0; /* no need to create if directory already exists */ + } + int r; +#if defined(_MSC_VER) + r = _mkdir(outDirName); + if (r || !UTIL_isDirectory(outDirName)) return 1; +#else + r = mkdir(outDirName, S_IRWXU | S_IRWXG | S_IRWXO); /* dir has all permissions */ + if (r || !UTIL_isDirectory(outDirName)) return 1; +#endif + return 0; +} + +void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles, + const char* outDirName, char** dstFilenameTable) +{ + unsigned u; + char c; + c = '/'; + + /* duplicate source file table */ + for (u = 0; u < nbFiles; ++u) { + char* filename; + char* finalPath; + size_t finalPathLen; + finalPathLen = strlen(outDirName); + filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ + finalPathLen += strlen(filename); + dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); + strcpy(dstFilenameTable[u], outDirName); + strcat(dstFilenameTable[u], filename); + } +} + +void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { + unsigned u; + for (u = 0; u < nbFiles; ++u) + free(dstDirTable[u]); + free((void*)dstDirTable); +} + int UTIL_isSameFile(const char* file1, const char* file2) { #if defined(_MSC_VER) diff --git a/programs/util.h b/programs/util.h index d6e5bb550..1c1273262 100644 --- a/programs/util.h +++ b/programs/util.h @@ -127,8 +127,12 @@ int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); int UTIL_setFileStat(const char* filename, stat_t* statbuf); U32 UTIL_isDirectory(const char* infilename); +int UTIL_createDir(const char* outDirName); int UTIL_getFileStat(const char* infilename, stat_t* statbuf); int UTIL_isSameFile(const char* file1, const char* file2); +void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx, + const char* outDirName, char** dstFilenameTable); +void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 98df728a9..2d3254ae5 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -118,6 +118,7 @@ static int usage(const char* programName) #endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); + DISPLAY( " -O directory: result(s) stored into `directory`, creates one if non-existent \n"); DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); @@ -562,6 +563,7 @@ int main(int argCount, const char* argv[]) adaptMax = MAXCLEVEL, rsyncable = 0, nextArgumentIsOutFileName = 0, + nextArgumentIsOutDirName = 0, nextArgumentIsMaxDict = 0, nextArgumentIsDictID = 0, nextArgumentsAreFiles = 0, @@ -583,9 +585,11 @@ int main(int argCount, const char* argv[]) unsigned recursive = 0; unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ + char** dstFilenameTable = (char**)malloc(argCount * sizeof(char*)); unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; + const char* outDirName = NULL; const char* dictFileName = NULL; const char* suffix = ZSTD_EXTENSION; unsigned maxDictSize = g_defaultMaxDictSize; @@ -853,6 +857,9 @@ int main(int argCount, const char* argv[]) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; + /* destination directory name */ + case 'O': nextArgumentIsOutDirName=1; lastCommand=1; argument++; break; + /* limit decompression memory */ case 'M': argument++; @@ -965,6 +972,13 @@ int main(int argCount, const char* argv[]) continue; } + if (nextArgumentIsOutDirName) { + nextArgumentIsOutDirName = 0; + lastCommand = 0; + outDirName = argument; + continue; + } + /* add filename to list */ filenameTable[filenameIdx++] = argument; } @@ -1163,10 +1177,20 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; + if (outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) DISPLAY("Directory creation unsuccessful \n"); + + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } + } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); From 6beb3c0159a60a6c49334f24e32ae92df5dbba4d Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 5 Sep 2019 17:56:24 -0700 Subject: [PATCH 02/17] added decompression support --- programs/fileio.c | 30 ++++++++++++++++++++++++++---- programs/fileio.h | 5 +++++ programs/util.c | 10 +++++++--- programs/zstdcli.c | 17 +++++++---------- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index f25771864..c8a971c27 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1422,7 +1422,6 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN const char* suffix, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { - printf("compressing multiple...\n"); int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); @@ -1457,7 +1456,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + /*UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);*/ return error; } @@ -2240,14 +2239,24 @@ FIO_determineDstName(const char* srcFileName) int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, - const char* srcNamesTable[], unsigned nbFiles, + const char** srcNamesTable, unsigned nbFiles, + const char* outDirName, char** dstFileNamesTable, const char* outFileName, const char* dictFileName) { int error = 0; dRess_t ress = FIO_createDResources(prefs, dictFileName); - if (outFileName) { + if (outDirName != NULL) { /* output into a particular folder */ + unsigned u; + for (u = 0; u < nbFiles; ++u) { + const char* const srcFileName = srcNamesTable[u]; + const char* const dstFileName = FIO_determineDstName(dstFileNamesTable[u]); + if (dstFileName == NULL) { error=1; continue; } + + error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); + } + } else if (outFileName) { unsigned u; ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); @@ -2268,10 +2277,23 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); + /* UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); */ return error; } +void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) + DISPLAY("Directory creation unsuccessful \n"); + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } +} /* ************************************************************************** * .zst file info (--list command) diff --git a/programs/fileio.h b/programs/fileio.h index 1ed6d1538..1b435c5f3 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -114,9 +114,14 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN @return : nb of missing or skipped files */ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char** srcNamesTable, unsigned nbFiles, + const char* outDirName, char** dstFilenameTable, const char* outFileName, const char* dictFileName); +void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName); + /*-************************************* * Advanced stuff (should actually be hosted elsewhere) diff --git a/programs/util.c b/programs/util.c index e8b599aaa..448a78899 100644 --- a/programs/util.c +++ b/programs/util.c @@ -108,6 +108,7 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles unsigned u; char c; c = '/'; + printf("NBFILE: %u\n", nbFiles); /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { @@ -120,14 +121,17 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); strcpy(dstFilenameTable[u], outDirName); strcat(dstFilenameTable[u], filename); + printf("%s %s\n", filenameTable[u], dstFilenameTable[u]); } } void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { unsigned u; - for (u = 0; u < nbFiles; ++u) - free(dstDirTable[u]); - free((void*)dstDirTable); + for (u = 0; u < nbFiles; ++u) { + if (dstDirTable[u] != NULL) + free(dstDirTable[u]); + } + if (dstDirTable != NULL) free((void*)dstDirTable); } int UTIL_isSameFile(const char* file1, const char* file2) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 2d3254ae5..7c00b3d71 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1177,16 +1177,9 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; - if (outDirName) { - int dirResult; - dirResult = UTIL_createDir(outDirName); - if (dirResult) DISPLAY("Directory creation unsuccessful \n"); + if (outDirName) + FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); - UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); - if (outFileName) { - outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ - } - } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else @@ -1205,10 +1198,14 @@ int main(int argCount, const char* argv[]) } } FIO_setMemLimit(prefs, memLimit); + + if (outDirName) + FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else - operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName); #else DISPLAY("Decompression not supported \n"); #endif From a9c807a9489cc1c313ab027c8206fb95eb1adc0e Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 6 Sep 2019 10:17:04 -0700 Subject: [PATCH 03/17] kill memory leaks, cleanup, fix some dumb bugs --- programs/fileio.c | 7 ++----- programs/util.c | 14 ++++++-------- programs/zstdcli.c | 11 +++++++---- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c8a971c27..194ea85c4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1390,10 +1390,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) { static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - size_t const sfnSize = strlen(srcFileName); size_t const suffixSize = strlen(suffix); - if (dfnbCapacity <= sfnSize+suffixSize+1) { /* resize buffer for dstName */ free(dstFileNameBuffer); @@ -1405,7 +1403,6 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) assert(dstFileNameBuffer != NULL); memcpy(dstFileNameBuffer, srcFileName, sfnSize); memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */); - return dstFileNameBuffer; } @@ -1456,7 +1453,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - /*UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);*/ + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } @@ -2277,7 +2274,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); - /* UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); */ + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } diff --git a/programs/util.c b/programs/util.c index 448a78899..3b134f9cd 100644 --- a/programs/util.c +++ b/programs/util.c @@ -87,11 +87,12 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } -int UTIL_createDir(const char* outDirName) { - if (UTIL_isDirectory(outDirName)) { - return 0; /* no need to create if directory already exists */ - } +int UTIL_createDir(const char* outDirName) +{ int r; + if (UTIL_isDirectory(outDirName)) + return 0; /* no need to create if directory already exists */ + #if defined(_MSC_VER) r = _mkdir(outDirName); if (r || !UTIL_isDirectory(outDirName)) return 1; @@ -108,20 +109,17 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles unsigned u; char c; c = '/'; - printf("NBFILE: %u\n", nbFiles); /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { char* filename; - char* finalPath; size_t finalPathLen; finalPathLen = strlen(outDirName); filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ finalPathLen += strlen(filename); - dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); + dstFilenameTable[u] = (char*) malloc((finalPathLen+5) * sizeof(char)); /* extra 1 bit for \0, extra 4 for .zst if compressing*/ strcpy(dstFilenameTable[u], outDirName); strcat(dstFilenameTable[u], filename); - printf("%s %s\n", filenameTable[u], dstFilenameTable[u]); } } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7c00b3d71..9029f6344 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -585,7 +585,7 @@ int main(int argCount, const char* argv[]) unsigned recursive = 0; unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ - char** dstFilenameTable = (char**)malloc(argCount * sizeof(char*)); + char** dstFilenameTable; unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; @@ -1177,8 +1177,10 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; - if (outDirName) + if (outDirName) { + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); @@ -1199,9 +1201,10 @@ int main(int argCount, const char* argv[]) } FIO_setMemLimit(prefs, memLimit); - if (outDirName) + if (outDirName) { + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); - + } if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else From 30bff50e06d84d1d426fee62acd34223abc69f1b Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 6 Sep 2019 11:10:53 -0700 Subject: [PATCH 04/17] fixes for tests and segfault --- programs/fileio.c | 22 ++++++---------------- programs/fileio.h | 4 ---- programs/util.c | 14 ++++++++++++++ programs/util.h | 3 +++ programs/zstdcli.c | 5 +++-- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 194ea85c4..cc8809f94 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1453,7 +1453,9 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + if (outDirName) + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + return error; } @@ -2274,24 +2276,12 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + if (outDirName) + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + return error; } -void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName) { - int dirResult; - dirResult = UTIL_createDir(outDirName); - if (dirResult) - DISPLAY("Directory creation unsuccessful \n"); - - UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); - if (outFileName) { - outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ - } -} - /* ************************************************************************** * .zst file info (--list command) ***************************************************************************/ diff --git a/programs/fileio.h b/programs/fileio.h index 1b435c5f3..40cd8985f 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -118,10 +118,6 @@ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char* outFileName, const char* dictFileName); -void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName); - /*-************************************* * Advanced stuff (should actually be hosted elsewhere) diff --git a/programs/util.c b/programs/util.c index 3b134f9cd..f00bfd608 100644 --- a/programs/util.c +++ b/programs/util.c @@ -123,6 +123,20 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles } } +void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) + UTIL_DISPLAYLEVEL(1, "Directory creation unsuccessful\n"); + + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } +} + void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { unsigned u; for (u = 0; u < nbFiles; ++u) { diff --git a/programs/util.h b/programs/util.h index 1c1273262..9615504ca 100644 --- a/programs/util.h +++ b/programs/util.h @@ -133,6 +133,9 @@ int UTIL_isSameFile(const char* file1, const char* file2); void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx, const char* outDirName, char** dstFilenameTable); void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); +void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9029f6344..606c162f7 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1178,8 +1178,9 @@ int main(int argCount, const char* argv[]) if (adaptMax < cLevel) cLevel = adaptMax; if (outDirName) { + printf("ok\n"); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); } if ((filenameIdx==1) && outFileName) @@ -1203,7 +1204,7 @@ int main(int argCount, const char* argv[]) if (outDirName) { dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); } if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); From 62616c4d90557ab070ddd7677840e882cf0e34b9 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 6 Sep 2019 13:20:50 -0700 Subject: [PATCH 05/17] fixes for windows compilation --- programs/util.c | 7 +++++-- programs/zstdcli.c | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/programs/util.c b/programs/util.c index f00bfd608..920601fba 100644 --- a/programs/util.c +++ b/programs/util.c @@ -20,6 +20,9 @@ extern "C" { #include #include +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) +#include /* needed for _mkdir in windows */ +#endif int UTIL_fileExist(const char* filename) { @@ -93,7 +96,7 @@ int UTIL_createDir(const char* outDirName) if (UTIL_isDirectory(outDirName)) return 0; /* no need to create if directory already exists */ -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) r = _mkdir(outDirName); if (r || !UTIL_isDirectory(outDirName)) return 1; #else @@ -112,7 +115,7 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { - char* filename; + const char* filename; size_t finalPathLen; finalPathLen = strlen(outDirName); filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 606c162f7..5fc6b8aa8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1181,6 +1181,8 @@ int main(int argCount, const char* argv[]) printf("ok\n"); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + } else { + dstFilenameTable = NULL; } if ((filenameIdx==1) && outFileName) @@ -1205,7 +1207,10 @@ int main(int argCount, const char* argv[]) if (outDirName) { dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + } else { + dstFilenameTable = NULL; } + if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else From f80437c58665294083c27ebba519b303352fa437 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 11:08:20 -0400 Subject: [PATCH 06/17] Add support for --output-dir-flat New flag to specify output directory destination for multiple files. --- build/LICENSE | 0 programs/fileio.c | 2 +- programs/util.c | 94 ++++++++++++++++++++++++++++++---------------- programs/util.h | 11 +++--- programs/zstdcli.c | 27 ++++++++----- tests/playTests.sh | 19 +++++++++- 6 files changed, 103 insertions(+), 50 deletions(-) create mode 100644 build/LICENSE diff --git a/build/LICENSE b/build/LICENSE new file mode 100644 index 000000000..e69de29bb diff --git a/programs/fileio.c b/programs/fileio.c index cc8809f94..c0300f6a0 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1453,7 +1453,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - if (outDirName) + if (dstFileNamesTable) UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; diff --git a/programs/util.c b/programs/util.c index 920601fba..e5c937864 100644 --- a/programs/util.c +++ b/programs/util.c @@ -90,53 +90,81 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } -int UTIL_createDir(const char* outDirName) -{ - int r; - if (UTIL_isDirectory(outDirName)) - return 0; /* no need to create if directory already exists */ +int UTIL_compareStr(const void *p1, const void *p2) { + return strcmp(* (char * const *) p1, * (char * const *) p2); +} -#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) - r = _mkdir(outDirName); - if (r || !UTIL_isDirectory(outDirName)) return 1; -#else - r = mkdir(outDirName, S_IRWXU | S_IRWXG | S_IRWXO); /* dir has all permissions */ - if (r || !UTIL_isDirectory(outDirName)) return 1; -#endif +int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles) { + char** dstFilenameTableSorted; + char* prevElem; + unsigned u; + + dstFilenameTableSorted = (char**) malloc(sizeof(char*) * nbFiles); + if (!dstFilenameTableSorted) { + UTIL_DISPLAYLEVEL(1, "Unable to malloc new str array, not checking for name collisions\n"); + return 1; + } + + for (u = 0; u < nbFiles; ++u) { + dstFilenameTableSorted[u] = dstFilenameTable[u]; + } + qsort(dstFilenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + prevElem = dstFilenameTableSorted[0]; + for (u = 1; u < nbFiles; ++u) { + if (strcmp(prevElem, dstFilenameTableSorted[u]) == 0) { + UTIL_DISPLAYLEVEL(1, "WARNING: Two files have same filename as source : %s\n", prevElem); + } + prevElem = dstFilenameTableSorted[u]; + } + + free(dstFilenameTableSorted); return 0; } -void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles, - const char* outDirName, char** dstFilenameTable) +void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, + const unsigned nbFiles, const char* outDirName, const int compressing) { unsigned u; - char c; - c = '/'; + const char* c; + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + c = "\\"; + #else + c = "/"; + #endif - /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { - const char* filename; + char* filename, *filenameBegin; size_t finalPathLen; finalPathLen = strlen(outDirName); - filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ + filenameBegin = strrchr(filenameTable[u], c[0]); + if (filenameBegin == NULL) { + filename = strdup(filenameTable[u]); + } else { + filename = strdup(filenameBegin+1); + } + finalPathLen += strlen(filename); - dstFilenameTable[u] = (char*) malloc((finalPathLen+5) * sizeof(char)); /* extra 1 bit for \0, extra 4 for .zst if compressing*/ + dstFilenameTable[u] = compressing ? + (char*) malloc((finalPathLen+6) * sizeof(char)) /* 4 more bytes for .zst suffix */ + : (char*) malloc((finalPathLen+2) * sizeof(char)); + if (!dstFilenameTable[u]) { + UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n"); + continue; + } + strcpy(dstFilenameTable[u], outDirName); - strcat(dstFilenameTable[u], filename); - } -} + if (outDirName[strlen(outDirName)-1] == c[0]) { + strcat(dstFilenameTable[u], filename); + } else { + strcat(dstFilenameTable[u], c); + strcat(dstFilenameTable[u], filename); + } -void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName) { - int dirResult; - dirResult = UTIL_createDir(outDirName); - if (dirResult) - UTIL_DISPLAYLEVEL(1, "Directory creation unsuccessful\n"); + free(filename); + } - UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); - if (outFileName) { - outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + if (UTIL_checkFilenameCollisions(dstFilenameTable, nbFiles)) { + UTIL_DISPLAYLEVEL(1, "Checking for filename collisions failed"); } } diff --git a/programs/util.h b/programs/util.h index 9615504ca..e90b251de 100644 --- a/programs/util.h +++ b/programs/util.h @@ -127,15 +127,14 @@ int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); int UTIL_setFileStat(const char* filename, stat_t* statbuf); U32 UTIL_isDirectory(const char* infilename); -int UTIL_createDir(const char* outDirName); int UTIL_getFileStat(const char* infilename, stat_t* statbuf); int UTIL_isSameFile(const char* file1, const char* file2); -void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx, - const char* outDirName, char** dstFilenameTable); +int UTIL_compareStr(const void *p1, const void *p2); +int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles); +/* Populates dstFilenameTable using outDirName concatenated with entries from filenameTable */ +void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, const unsigned nbFiles, + const char* outDirName, const int compressing); void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); -void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 5fc6b8aa8..6b761ffa6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -118,7 +118,6 @@ static int usage(const char* programName) #endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); - DISPLAY( " -O directory: result(s) stored into `directory`, creates one if non-existent \n"); DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); @@ -137,6 +136,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -l : print information about zstd compressed files \n"); + DISPLAY( " --output-dir-flat directory: results stored into `directory` top level \n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); @@ -690,6 +690,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; } if (!strcmp(argument, "--adapt")) { adapt = 1; continue; } if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; } if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } @@ -856,9 +857,6 @@ int main(int argCount, const char* argv[]) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; - - /* destination directory name */ - case 'O': nextArgumentIsOutDirName=1; lastCommand=1; argument++; break; /* limit decompression memory */ case 'M': @@ -1178,9 +1176,14 @@ int main(int argCount, const char* argv[]) if (adaptMax < cLevel) cLevel = adaptMax; if (outDirName) { - printf("ok\n"); - dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + if (UTIL_isDirectory(outDirName)) { + DISPLAY("Output of files will be in directory: %s\n", outDirName); + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); + UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1); + } else { + DISPLAY("%s is not a directory!\n", outDirName); + CLEAN_RETURN(1); + } } else { dstFilenameTable = NULL; } @@ -1205,8 +1208,14 @@ int main(int argCount, const char* argv[]) FIO_setMemLimit(prefs, memLimit); if (outDirName) { - dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + if (UTIL_isDirectory(outDirName)) { + DISPLAY("Output of files will be in directory: %s\n", outDirName); + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); + UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1); + } else { + DISPLAY("%s is not a directory!\n", outDirName); + CLEAN_RETURN(1); + } } else { dstFilenameTable = NULL; } diff --git a/tests/playTests.sh b/tests/playTests.sh index 19fc514f6..8851da2f5 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -264,6 +264,24 @@ if [ "$?" -eq 139 ]; then fi rm tmp* +println "test : compress multiple files into an output directory, --output-dir-flat" +println henlo > tmp1 +mkdir tmpInputTestDir +mkdir tmpInputTestDir/we +mkdir tmpInputTestDir/we/must +mkdir tmpInputTestDir/we/must/go +mkdir tmpInputTestDir/we/must/go/deeper +println cool > tmpInputTestDir/we/must/go/deeper/tmp2 +mkdir tmpOutDir +$ZSTD tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir +test -f tmpOutDir/tmp1.zst +test -f tmpOutDir/tmp2.zst +println "test : decompress multiple files into an output directory, --output-dir-flat" +mkdir tmpOutDirDecomp +$ZSTD tmpOutDir/ -r -d --output-dir-flat tmpOutDirDecomp +test -f tmpOutDirDecomp/tmp2 +test -f tmpOutDirDecomp/tmp1 +rm -rf tmp* println "\n===> Advanced compression parameters " println "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" @@ -407,7 +425,6 @@ ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) println "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" - println "\n===> stream-size mode" ./datagen -g11000 > tmp From 1e4d602412241c142b7f57e1d7f4995e12a003e0 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 11:11:35 -0400 Subject: [PATCH 07/17] Modified message --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6b761ffa6..63e71e7e6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -136,7 +136,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -l : print information about zstd compressed files \n"); - DISPLAY( " --output-dir-flat directory: results stored into `directory` top level \n"); + DISPLAY( " --output-dir-flat directory: results stored into `directory`. Filename collisions mean first file will be compressed. With -f, the last file will be compressed.\n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); From 147b7614544e411e2087c3e696f4c1be046986cf Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 11:18:14 -0400 Subject: [PATCH 08/17] Removed strdup() dependency --- programs/util.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/util.c b/programs/util.c index e5c937864..083eb12e0 100644 --- a/programs/util.c +++ b/programs/util.c @@ -138,9 +138,11 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena finalPathLen = strlen(outDirName); filenameBegin = strrchr(filenameTable[u], c[0]); if (filenameBegin == NULL) { - filename = strdup(filenameTable[u]); + filename = (char*) malloc((strlen(filenameTable[u])+1) * sizeof(char)); + strcpy(filename, filenameTable[u]); } else { - filename = strdup(filenameBegin+1); + filename = (char*) malloc((strlen(filenameBegin+1)) * sizeof(char)); + strcpy(filename, filenameBegin+1); } finalPathLen += strlen(filename); From b93f1b2a30c55cc0d5938c71f33b9e83f28d2d7a Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 11:29:34 -0400 Subject: [PATCH 09/17] CI Tests fix --- programs/util.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/util.c b/programs/util.c index 083eb12e0..aa769567e 100644 --- a/programs/util.c +++ b/programs/util.c @@ -133,7 +133,8 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena #endif for (u = 0; u < nbFiles; ++u) { - char* filename, *filenameBegin; + char* filename; + const char* filenameBegin; size_t finalPathLen; finalPathLen = strlen(outDirName); filenameBegin = strrchr(filenameTable[u], c[0]); From c763457e0a4d70b6912bc14c5908a38c3239a0bf Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 15:30:24 -0400 Subject: [PATCH 10/17] Static analyze fix --- programs/util.c | 7 +++---- programs/util.h | 2 +- programs/zstdcli.c | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/programs/util.c b/programs/util.c index aa769567e..83fa70bae 100644 --- a/programs/util.c +++ b/programs/util.c @@ -122,7 +122,7 @@ int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles) { } void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, - const unsigned nbFiles, const char* outDirName, const int compressing) + const unsigned nbFiles, const char* outDirName) { unsigned u; const char* c; @@ -147,11 +147,10 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena } finalPathLen += strlen(filename); - dstFilenameTable[u] = compressing ? - (char*) malloc((finalPathLen+6) * sizeof(char)) /* 4 more bytes for .zst suffix */ - : (char*) malloc((finalPathLen+2) * sizeof(char)); + dstFilenameTable[u] = (char*) malloc((finalPathLen+2) * sizeof(char)); if (!dstFilenameTable[u]) { UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n"); + free(filename); continue; } diff --git a/programs/util.h b/programs/util.h index e90b251de..558e7a9c6 100644 --- a/programs/util.h +++ b/programs/util.h @@ -133,7 +133,7 @@ int UTIL_compareStr(const void *p1, const void *p2); int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles); /* Populates dstFilenameTable using outDirName concatenated with entries from filenameTable */ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, const unsigned nbFiles, - const char* outDirName, const int compressing); + const char* outDirName); void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); U32 UTIL_isLink(const char* infilename); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 63e71e7e6..d9e1be66a 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1179,7 +1179,7 @@ int main(int argCount, const char* argv[]) if (UTIL_isDirectory(outDirName)) { DISPLAY("Output of files will be in directory: %s\n", outDirName); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1); + UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName); } else { DISPLAY("%s is not a directory!\n", outDirName); CLEAN_RETURN(1); @@ -1211,7 +1211,7 @@ int main(int argCount, const char* argv[]) if (UTIL_isDirectory(outDirName)) { DISPLAY("Output of files will be in directory: %s\n", outDirName); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1); + UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName); } else { DISPLAY("%s is not a directory!\n", outDirName); CLEAN_RETURN(1); From 4dc604cab8f5d5e89454d2afeffe32f2001c41aa Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 18:34:42 -0400 Subject: [PATCH 11/17] Addressed comments on malloc --- programs/fileio.c | 6 ------ programs/util.c | 12 ++++++++++-- programs/zstdcli.c | 5 +++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c0300f6a0..6006af01b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1453,9 +1453,6 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - if (dstFileNamesTable) - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); - return error; } @@ -2276,9 +2273,6 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); - if (outDirName) - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); - return error; } diff --git a/programs/util.c b/programs/util.c index 83fa70bae..c12a31887 100644 --- a/programs/util.c +++ b/programs/util.c @@ -140,14 +140,22 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena filenameBegin = strrchr(filenameTable[u], c[0]); if (filenameBegin == NULL) { filename = (char*) malloc((strlen(filenameTable[u])+1) * sizeof(char)); + if (!filename) { + UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n"); + continue; + } strcpy(filename, filenameTable[u]); } else { - filename = (char*) malloc((strlen(filenameBegin+1)) * sizeof(char)); + filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char)); + if (!filename) { + UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n"); + continue; + } strcpy(filename, filenameBegin+1); } finalPathLen += strlen(filename); - dstFilenameTable[u] = (char*) malloc((finalPathLen+2) * sizeof(char)); + dstFilenameTable[u] = (char*) malloc((finalPathLen+3) * sizeof(char)); if (!dstFilenameTable[u]) { UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n"); free(filename); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d9e1be66a..91e67a2d3 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1192,6 +1192,9 @@ int main(int argCount, const char* argv[]) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); + + if (dstFilenameTable) + UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx); #else (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); @@ -1224,6 +1227,8 @@ int main(int argCount, const char* argv[]) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName); + if (dstFilenameTable) + UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx); #else DISPLAY("Decompression not supported \n"); #endif From 64bc441d7d06b267aba52cfdae681a5582d4cfe7 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 3 Oct 2019 13:53:04 -0400 Subject: [PATCH 12/17] Now constructs final destination path without allocating new table --- programs/fileio.c | 184 +++++++++++++++++++++++++++++++++++---------- programs/fileio.h | 14 ++-- programs/util.c | 93 ----------------------- programs/util.h | 5 -- programs/zstdcli.c | 37 +-------- 5 files changed, 157 insertions(+), 176 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 6006af01b..9988e2d6a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -628,6 +628,104 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) return (size_t)fileSize; } + + +/* FIO_checkFilenameCollisions() : + * Checks for and warns if there are any files that would have the same output path + */ +int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { + const char** filenameTableSorted; + const char* c, *prevElem; + char* filename; + unsigned u; + + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + c = "\\"; + #else + c = "/"; + #endif + + filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles); + if (!filenameTableSorted) { + DISPLAY("Unable to malloc new str array, not checking for name collisions\n"); + return 1; + } + + for (u = 0; u < nbFiles; ++u) { + filename = strrchr(filenameTable[u], c[0]); + if (filename == NULL) { + filenameTableSorted[u] = filenameTable[u]; + } else { + filenameTableSorted[u] = filename+1; + } + } + + qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + prevElem = filenameTableSorted[0]; + for (u = 1; u < nbFiles; ++u) { + if (strcmp(prevElem, filenameTableSorted[u]) == 0) { + DISPLAY("WARNING: Two files have same filename: %s\n", prevElem); + } + prevElem = filenameTableSorted[u]; + } + + free(filenameTableSorted); + return 0; +} + +/* FIO_determineDstFilenameOutdir() : + * Takes a source file name and specified output directory, and + * allocates memory for and returns a pointer to final path. + * This function never returns an error (it may abort() in case of pb) + */ +static char* +FIO_determineDstFilenameOutdir(const char* srcFilename, const char* outDirName, const size_t suffixLen) +{ + const char* c, *filenameBegin; + char* filename, *result; + size_t finalPathLen; + + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + c = "\\"; + #else + c = "/"; + #endif + + finalPathLen = strlen(outDirName); + filenameBegin = strrchr(srcFilename, c[0]); + if (filenameBegin == NULL) { + filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char)); + if (!filename) { + EXM_THROW(30, "zstd: %s", strerror(errno)); + } + strcpy(filename, srcFilename); + } else { + filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char)); + if (!filename) { + EXM_THROW(30, "zstd: %s", strerror(errno)); + } + strcpy(filename, filenameBegin+1); + } + + finalPathLen += strlen(filename); + result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char)); + if (!result) { + free(filename); + EXM_THROW(30, "zstd: %s", strerror(errno)); + } + + strcpy(result, outDirName); + if (outDirName[strlen(outDirName)-1] == c[0]) { + strcat(result, filename); + } else { + strcat(result, c); + strcat(result, filename); + } + + free(filename); + return result; +} + #ifndef ZSTD_NOCOMPRESS /* ********************************************************************** @@ -1379,19 +1477,25 @@ int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName, return result; } - /* FIO_determineCompressedName() : * create a destination filename for compressed srcFileName. * @return a pointer to it. * This function never returns an error (it may abort() in case of pb) */ static const char* -FIO_determineCompressedName(const char* srcFileName, const char* suffix) +FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix) { static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - size_t const sfnSize = strlen(srcFileName); + char* outDirFilename = NULL; + size_t sfnSize = strlen(srcFileName); size_t const suffixSize = strlen(suffix); + if (outDirName) { + outDirFilename = FIO_determineDstFilenameOutdir(srcFileName, outDirName, suffixSize); + sfnSize = strlen(outDirFilename); + assert(outDirFilename != NULL); + } + if (dfnbCapacity <= sfnSize+suffixSize+1) { /* resize buffer for dstName */ free(dstFileNameBuffer); @@ -1399,9 +1503,16 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) dstFileNameBuffer = (char*)malloc(dfnbCapacity); if (!dstFileNameBuffer) { EXM_THROW(30, "zstd: %s", strerror(errno)); - } } + } + } assert(dstFileNameBuffer != NULL); - memcpy(dstFileNameBuffer, srcFileName, sfnSize); + + if (outDirFilename) { + memcpy(dstFileNameBuffer, outDirFilename, sfnSize); + free(outDirFilename); + } else { + memcpy(dstFileNameBuffer, srcFileName, sfnSize); + } memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */); return dstFileNameBuffer; } @@ -1414,24 +1525,17 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) * or into a destination folder (specified with -O) */ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, - const char* outDirName, char** dstFileNamesTable, - unsigned nbFiles, const char* outFileName, - const char* suffix, const char* dictFileName, - int compressionLevel, ZSTD_compressionParameters comprParams) + const char* outDirName, unsigned nbFiles, + const char* outFileName, const char* suffix, + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters comprParams) { int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); /* init */ assert(outFileName != NULL || suffix != NULL); - if (outDirName != NULL) { /* output into a particular folder */ - unsigned u; - for (u = 0; u < nbFiles; ++u) { - const char* const srcFileName = inFileNamesTable[u]; - const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix); - error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); - } - } else if (outFileName != NULL) { /* output into a single destination (stdout typically) */ + if (outFileName != NULL) { /* output into a single destination (stdout typically) */ ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; @@ -1448,9 +1552,11 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN unsigned u; for (u=0; u= 1 */ - char** dstFilenameTable; unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; @@ -1175,26 +1174,10 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; - if (outDirName) { - if (UTIL_isDirectory(outDirName)) { - DISPLAY("Output of files will be in directory: %s\n", outDirName); - dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName); - } else { - DISPLAY("%s is not a directory!\n", outDirName); - CLEAN_RETURN(1); - } - } else { - dstFilenameTable = NULL; - } - if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); - - if (dstFilenameTable) - UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx); + operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); @@ -1209,26 +1192,10 @@ int main(int argCount, const char* argv[]) } } FIO_setMemLimit(prefs, memLimit); - - if (outDirName) { - if (UTIL_isDirectory(outDirName)) { - DISPLAY("Output of files will be in directory: %s\n", outDirName); - dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName); - } else { - DISPLAY("%s is not a directory!\n", outDirName); - CLEAN_RETURN(1); - } - } else { - dstFilenameTable = NULL; - } - if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else - operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName); - if (dstFilenameTable) - UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx); + operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName); #else DISPLAY("Decompression not supported \n"); #endif From 6b81bfb257523f4638e0e1e6005d30776cdffb6e Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 3 Oct 2019 15:23:49 -0400 Subject: [PATCH 13/17] Changed function name, fix test --- programs/fileio.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9988e2d6a..6f1a2d678 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -634,9 +634,7 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) * Checks for and warns if there are any files that would have the same output path */ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { - const char** filenameTableSorted; - const char* c, *prevElem; - char* filename; + const char **filenameTableSorted, *c, *prevElem, *filename; unsigned u; #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ @@ -673,13 +671,13 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { return 0; } -/* FIO_determineDstFilenameOutdir() : +/* FIO_createFilename_fromOutDir() : * Takes a source file name and specified output directory, and * allocates memory for and returns a pointer to final path. * This function never returns an error (it may abort() in case of pb) */ static char* -FIO_determineDstFilenameOutdir(const char* srcFilename, const char* outDirName, const size_t suffixLen) +FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, const size_t suffixLen) { const char* c, *filenameBegin; char* filename, *result; @@ -1491,7 +1489,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con size_t sfnSize = strlen(srcFileName); size_t const suffixSize = strlen(suffix); if (outDirName) { - outDirFilename = FIO_determineDstFilenameOutdir(srcFileName, outDirName, suffixSize); + outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, suffixSize); sfnSize = strlen(outDirFilename); assert(outDirFilename != NULL); } @@ -2279,7 +2277,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) { static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - char* dstFilenameOutDir = NULL; + char* outDirFilename = NULL; size_t sfnSize = strlen(srcFileName); size_t suffixSize; @@ -2321,9 +2319,9 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) return NULL; } if (outDirName) { - dstFilenameOutDir = FIO_determineDstFilenameOutdir(srcFileName, outDirName, 0); - sfnSize = strlen(dstFilenameOutDir); - assert(dstFilenameOutDir != NULL); + outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0); + sfnSize = strlen(outDirFilename); + assert(outDirFilename != NULL); } if (dfnbCapacity+suffixSize <= sfnSize+1) { @@ -2337,9 +2335,9 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) /* return dst name == src name truncated from suffix */ assert(dstFileNameBuffer != NULL); - if (dstFilenameOutDir) { - memcpy(dstFileNameBuffer, dstFilenameOutDir, sfnSize - suffixSize); - free(dstFilenameOutDir); + if (outDirFilename) { + memcpy(dstFileNameBuffer, outDirFilename, sfnSize - suffixSize); + free(outDirFilename); } else { memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize); } From f9b1da6f62b265f38603e916fd6621198b862a20 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 8 Oct 2019 00:15:28 -0400 Subject: [PATCH 14/17] Hopefully fix VS compiler warning --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 6f1a2d678..cc495b858 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -658,7 +658,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { } } - qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + qsort((const char**)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); prevElem = filenameTableSorted[0]; for (u = 1; u < nbFiles; ++u) { if (strcmp(prevElem, filenameTableSorted[u]) == 0) { @@ -667,7 +667,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { prevElem = filenameTableSorted[u]; } - free(filenameTableSorted); + free((const char**)filenameTableSorted); return 0; } From 332b5988d8f080aca68901e371b70451f7061fd8 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 8 Oct 2019 09:44:24 -0400 Subject: [PATCH 15/17] Suppress C4090 warning --- programs/fileio.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index cc495b858..9ac65abcb 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -658,7 +658,13 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { } } - qsort((const char**)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + /* Silence VS Compiler bug with warning C4090 */ + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) + #pragma warning (push) + #pragma warning (disable : 4090) + #endif + + qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); prevElem = filenameTableSorted[0]; for (u = 1; u < nbFiles; ++u) { if (strcmp(prevElem, filenameTableSorted[u]) == 0) { @@ -667,7 +673,11 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { prevElem = filenameTableSorted[u]; } - free((const char**)filenameTableSorted); + free(filenameTableSorted); + + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) + #pragma warning (pop) + #endif return 0; } From 6e406b55eeeaa3ce1175459fda4044cbf248e0d1 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 8 Oct 2019 09:54:59 -0400 Subject: [PATCH 16/17] Casting to void* to avoid C4090 warnings --- programs/fileio.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9ac65abcb..81633947f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -658,13 +658,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { } } - /* Silence VS Compiler bug with warning C4090 */ - #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) - #pragma warning (push) - #pragma warning (disable : 4090) - #endif - - qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); prevElem = filenameTableSorted[0]; for (u = 1; u < nbFiles; ++u) { if (strcmp(prevElem, filenameTableSorted[u]) == 0) { @@ -673,11 +667,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { prevElem = filenameTableSorted[u]; } - free(filenameTableSorted); - - #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) - #pragma warning (pop) - #endif + free((void*)filenameTableSorted); return 0; } From c5ebb370510b5a2c7fbeac332a2480f51e0c9070 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 9 Oct 2019 09:39:52 -0400 Subject: [PATCH 17/17] Only check for filename collisions when using --output-dir-flat --- programs/fileio.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 81633947f..39cc70b9c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1553,7 +1553,8 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN const char* const dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); } - FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles); + if (outDirName) + FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles); } FIO_freeCResources(ress); @@ -2375,7 +2376,8 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); } - FIO_checkFilenameCollisions(srcNamesTable ,nbFiles); + if (outDirName) + FIO_checkFilenameCollisions(srcNamesTable ,nbFiles); } FIO_freeDResources(ress);