From 0f2bff2faf0cb60c4fe5346f56a5b42558a8e1a9 Mon Sep 17 00:00:00 2001 From: Shashank Tavildar Date: Mon, 28 Oct 2019 18:21:47 -0700 Subject: [PATCH] Addressing comments, removing cyclic dependency with header file, updating tests --- programs/fileio.c | 36 ++++++++++++++++++++++++++---------- programs/fileio.h | 1 + programs/util.c | 14 ++++---------- programs/util.h | 18 +----------------- programs/zstdcli.c | 2 +- tests/playTests.sh | 3 +++ 6 files changed, 36 insertions(+), 38 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 7cbf0280e..e59fb80f6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -319,6 +319,8 @@ struct FIO_prefs_s { /* Computation resources preferences */ unsigned memLimit; int nbWorkers; + + int excludeCompressedFiles; }; @@ -359,6 +361,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->srcSizeHint = 0; ret->testMode = 0; ret->literalCompressionMode = ZSTD_lcm_auto; + ret->excludeCompressedFiles = 0; return ret; } @@ -402,6 +405,8 @@ void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) { prefs->nbWorkers = nbWorkers; } +void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; } + void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { if (blockSize && prefs->nbWorkers==0) DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); @@ -1425,6 +1430,18 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, return result; } +static const char *compressedFileExtensions[] = { + ZSTD_EXTENSION, + TZSTD_EXTENSION, + GZ_EXTENSION, + TGZ_EXTENSION, + LZMA_EXTENSION, + XZ_EXTENSION, + TXZ_EXTENSION, + LZ4_EXTENSION, + TLZ4_EXTENSION, + NULL +}; /*! FIO_compressFilename_srcFile() : * @return : 0 : compression completed correctly, @@ -1451,19 +1468,18 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, return 1; } + /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used + * YES => ZSTD will skip compression of the file and will return 0. + * NO => ZSTD will resume with compress operation. + */ + if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) { + DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName); + return 0; + } + ress.srcFile = FIO_openSrcFile(srcFileName); if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ - /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used - * YES => ZSTD will not compress the file. - * NO => ZSTD will resume with compress operation. - */ - if (g_excludeCompressedFiles && UTIL_isCompressedFile(srcFileName)) { /* precompressed file (--exclude-compressed). DO NOT COMPRESS */ - DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName); - fclose(ress.srcFile); - ress.srcFile = NULL; - return 0; - } result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); diff --git a/programs/fileio.h b/programs/fileio.h index af2c5d9d1..a7da089f6 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -93,6 +93,7 @@ void FIO_setLiteralCompressionMode( void FIO_setNoProgress(unsigned noProgress); void FIO_setNotificationLevel(int level); +void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles); /*-************************************* * Single File functions diff --git a/programs/util.c b/programs/util.c index f82079554..7212f7b81 100644 --- a/programs/util.c +++ b/programs/util.c @@ -331,24 +331,18 @@ YES => Skip the file (return 0) NO => return 1 */ -int UTIL_isCompressedFile(const char *inputName) +int UTIL_isCompressedFile(const char *inputName, const char *extensionList[]) { - return compareExtensions(inputName,g_compressedFileExtensions); -} - -int compareExtensions(const char* infilename, const char* extensionList[]) -{ - int i=0; - while(*extensionList != NULL) + while(*extensionList!=NULL) { - const char* ext = strstr(infilename,extensionList[i]); + const char* ext = strstr(inputName,*extensionList); if(ext) return 1; ++extensionList; - i++; } return 0; } + /* * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb). diff --git a/programs/util.h b/programs/util.h index deb70786a..deaea0323 100644 --- a/programs/util.h +++ b/programs/util.h @@ -39,7 +39,6 @@ extern "C" { #endif #include /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */ #include "mem.h" /* U32, U64 */ -#include "fileio.h" /*-************************************************************ * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW @@ -127,19 +126,6 @@ extern int g_utilDisplayLevel; typedef struct stat stat_t; #endif -int g_excludeCompressedFiles; -static const char *g_compressedFileExtensions[] = { - ZSTD_EXTENSION, - TZSTD_EXTENSION, - GZ_EXTENSION, - TGZ_EXTENSION, - LZMA_EXTENSION, - XZ_EXTENSION, - TXZ_EXTENSION, - LZ4_EXTENSION, - TLZ4_EXTENSION, - NULL -}; int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); @@ -148,9 +134,7 @@ U32 UTIL_isDirectory(const char* infilename); int UTIL_getFileStat(const char* infilename, stat_t* statbuf); int UTIL_isSameFile(const char* file1, const char* file2); int UTIL_compareStr(const void *p1, const void *p2); -int UTIL_isCompressedFile(const char* infilename); -int compareExtensions(const char* infilename, const char *extensionList[]); - +int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]); U32 UTIL_isFIFO(const char* infilename); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a704a1abd..5463c019f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -709,7 +709,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; } if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } - if (!strcmp(argument, "--exclude-compressed")) { g_excludeCompressedFiles = 1; continue; } + if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT if (longCommandWArg(&argument, "--train-cover")) { diff --git a/tests/playTests.sh b/tests/playTests.sh index ca286071d..cb703227b 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -224,6 +224,8 @@ sleep 5 ./datagen $size > precompressedFilterTestDir/input.7 ./datagen $size > precompressedFilterTestDir/input.8 $ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir +test ! -f input.5.zst.zst +test ! -f input.6.zst.zst file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s` file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s` if [[ $file2timestamp -ge $file1timestamp ]]; then @@ -232,6 +234,7 @@ else println "Test is not successful" fi println "Test completed" +sleep 5 println "test : file removal" $ZSTD -f --rm tmp