From 5b45ff4f1cd511a0bfba9ff28c8d73e03d9be4bd Mon Sep 17 00:00:00 2001
From: Bimba Shrestha <bimbashrestha@fb.com>
Date: Fri, 25 Oct 2019 11:32:38 -0700
Subject: [PATCH 01/27] Gating named file support on windows

---
 programs/fileio.c  | 12 ++++++++++++
 programs/zstdcli.c | 14 ++++++++++++++
 tests/playTests.sh |  4 ++++
 3 files changed, 30 insertions(+)

diff --git a/programs/fileio.c b/programs/fileio.c
index d45e4bbda..20868b9b8 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -513,12 +513,24 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
         return NULL;
     }
 
+#ifdef _MSC_VER
+
+    if (!UTIL_isRegularFile(srcFileName)) {
+        DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
+                        srcFileName);
+        return NULL;
+    }
+
+#else
+
     if (!UTIL_isRegularFile(srcFileName) && !UTIL_isFIFO(srcFileName)) {
         DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
                         srcFileName);
         return NULL;
     }
 
+#endif /* _MSC_VER */
+
     {   FILE* const f = fopen(srcFileName, "rb");
         if (f == NULL)
             DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 7a0fa6a25..66b7ae5d2 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -992,11 +992,25 @@ int main(int argCount, const char* argv[])
     if (!followLinks) {
         unsigned u;
         for (u=0, fileNamesNb=0; u<filenameIdx; u++) {
+
+#ifdef _MSC_VER
+
+            if (UTIL_isLink(filenameTable[u])) {
+                DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
+            } else {
+                filenameTable[fileNamesNb++] = filenameTable[u];
+            }
+
+#else
+
             if (UTIL_isLink(filenameTable[u]) && !UTIL_isFIFO(filenameTable[u])) {
                 DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
             } else {
                 filenameTable[fileNamesNb++] = filenameTable[u];
             }
+
+#endif /* _MSC_VER */
+
         }
         if (fileNamesNb == 0 && filenameIdx > 0)
             CLEAN_RETURN(1);
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 29ac1faa5..036318a54 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -1080,6 +1080,8 @@ test -f dictionary
 rm -f tmp* dictionary
 
 
+if [ "$isWindows" = false ] ; then
+
 println "\n===>  zstd fifo named pipe test "
 head -c 10 /dev/zero > tmp_original
 mkfifo named_pipe
@@ -1090,4 +1092,6 @@ $DIFF -s tmp_original tmp_decompressed
 rm -rf tmp*
 rm -rf named_pipe
 
+fi
+
 rm -f tmp*

From 8adecc73b0b9c01450c842b222491cfeacf5e308 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha <bimbashrestha@fb.com>
Date: Fri, 25 Oct 2019 12:04:54 -0700
Subject: [PATCH 02/27] Running playtests.sh on PRs too

---
 appveyor.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 87fa5c12d..3a4803363 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -264,5 +264,6 @@
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
   - if [%HOST%]==[mingw] (
       set "CC=%COMPILER%" &&
-      make check
+      make check &&
+      sh -e playTests.sh --test-large-data
     )

From 0b52d878b27b320a6398048db094794bac15b83a Mon Sep 17 00:00:00 2001
From: Bimba Shrestha <bimbashrestha@fb.com>
Date: Fri, 25 Oct 2019 14:06:50 -0700
Subject: [PATCH 03/27] Cleaning up gate and adding comment to flag

---
 programs/fileio.c  | 22 ++++++++--------------
 programs/util.c    |  4 +++-
 programs/zstdcli.c | 26 ++++++++------------------
 3 files changed, 19 insertions(+), 33 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 20868b9b8..48b6cb483 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -513,23 +513,17 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
         return NULL;
     }
 
-#ifdef _MSC_VER
 
-    if (!UTIL_isRegularFile(srcFileName)) {
-        DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
-                        srcFileName);
-        return NULL;
-    }
-
-#else
-
-    if (!UTIL_isRegularFile(srcFileName) && !UTIL_isFIFO(srcFileName)) {
-        DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
-                        srcFileName);
-        return NULL;
-    }
 
+    if (!UTIL_isRegularFile(srcFileName)
+#ifndef _MSC_VER
+        && !UTIL_isFIFO(srcFileName)
 #endif /* _MSC_VER */
+    ) {
+        DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
+                        srcFileName);
+        return NULL;
+    }
 
     {   FILE* const f = fopen(srcFileName, "rb");
         if (f == NULL)
diff --git a/programs/util.c b/programs/util.c
index 5f97b1cde..54aca5b55 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -115,6 +115,8 @@ int UTIL_isSameFile(const char* file1, const char* file2)
 #endif
 }
 
+#ifndef _MSC_VER
+/* Using this to distinguish named pipes */
 U32 UTIL_isFIFO(const char* infilename)
 {
 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */
@@ -126,7 +128,7 @@ U32 UTIL_isFIFO(const char* infilename)
     (void)infilename;
     return 0;
 }
-
+#endif
 
 U32 UTIL_isLink(const char* infilename)
 {
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 66b7ae5d2..fe77be22e 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -992,25 +992,15 @@ int main(int argCount, const char* argv[])
     if (!followLinks) {
         unsigned u;
         for (u=0, fileNamesNb=0; u<filenameIdx; u++) {
-
-#ifdef _MSC_VER
-
-            if (UTIL_isLink(filenameTable[u])) {
-                DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
-            } else {
-                filenameTable[fileNamesNb++] = filenameTable[u];
-            }
-
-#else
-
-            if (UTIL_isLink(filenameTable[u]) && !UTIL_isFIFO(filenameTable[u])) {
-                DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
-            } else {
-                filenameTable[fileNamesNb++] = filenameTable[u];
-            }
-
+            if (UTIL_isLink(filenameTable[u])
+#ifndef _MSC_VER
+                && !UTIL_isFIFO(filenameTable[u])
 #endif /* _MSC_VER */
-
+            ) {
+                DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
+            } else {
+                filenameTable[fileNamesNb++] = filenameTable[u];
+            }
         }
         if (fileNamesNb == 0 && filenameIdx > 0)
             CLEAN_RETURN(1);

From 857268b32c8efb99f1f40fab3534850f3871f007 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha <bimbashrestha@fb.com>
Date: Fri, 25 Oct 2019 15:15:28 -0700
Subject: [PATCH 04/27] Gating named pipe support in hedaer file

---
 programs/util.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/programs/util.h b/programs/util.h
index 3b15d9471..e79834693 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -135,7 +135,10 @@ U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 
+#ifndef _MSC_VER
 U32 UTIL_isFIFO(const char* infilename);
+#endif
+
 U32 UTIL_isLink(const char* infilename);
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))
 U64 UTIL_getFileSize(const char* infilename);

From 55ee7d56e4693057922cbd52b26b8ec1d4220bee Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Fri, 25 Oct 2019 15:49:11 -0700
Subject: [PATCH 05/27] Added --exclude-compressed flag feature that skips
 compression of precompressed files

---
 programs/fileio.c  |  7 ++++++-
 programs/util.c    | 21 +++++++++++++++++++++
 programs/util.h    |  4 ++++
 programs/zstdcli.c |  3 ++-
 tests/playTests.sh | 19 +++++++++++++++++++
 5 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 0365f1449..5172aa54c 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1453,7 +1453,12 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
 
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
-
+    if (g_excludeCompressedFiles && !UTIL_isPrecompressedFile(srcFileName)) {  /* precompressed file (--exclude-compressed). DO NOT COMPRESS */
+        DISPLAYLEVEL(4, "Precompressed file: %s \n", srcFileName);
+        fclose(ress.srcFile);
+        ress.srcFile = NULL;
+        return 0;
+    }
     result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);
diff --git a/programs/util.c b/programs/util.c
index 587058805..830f2039e 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -326,6 +326,27 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
 
 #endif /* #ifdef _WIN32 */
 
+/* Check if the file is precompressed (.zst, .lz4, .gz, .xz).
+YES => Skip the file (return 0)
+NO => return 1
+*/
+int UTIL_isPrecompressedFile(const char *inputName)
+{
+    return compareExtensions(inputName,compressedFileExtensions);
+}
+
+int compareExtensions(const char* infilename, const char extensionList[4][10])
+{
+   int i=0;
+   //char* ext = strchr(infilename, '.');
+   for(i=0;i<4;i++)
+   {
+     char* ext = strstr(infilename,extensionList[i]);
+     if(ext)
+        return 0;
+   }
+   return 1;
+}
 /*
  * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
  *                       and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
diff --git a/programs/util.h b/programs/util.h
index 71ba0d4fc..0da6255c0 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -127,6 +127,8 @@ extern int g_utilDisplayLevel;
     typedef struct stat stat_t;
 #endif
 
+int g_excludeCompressedFiles;
+static const char compressedFileExtensions[4][10] = {".zst",".gz",".xz",".lz4"};
 
 int UTIL_fileExist(const char* filename);
 int UTIL_isRegularFile(const char* infilename);
@@ -135,6 +137,8 @@ U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 int UTIL_compareStr(const void *p1, const void *p2);
+int UTIL_isPrecompressedFile(const char* infilename);
+int compareExtensions(const char* infilename, const char extensionList[4][10]);
 
 U32 UTIL_isFIFO(const char* infilename);
 U32 UTIL_isLink(const char* infilename);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 3d5c4280e..11116afc0 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -118,6 +118,7 @@ static int usage(const char* programName)
 #endif
     DISPLAY( " -D file: use `file` as Dictionary \n");
     DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
+    DISPLAY( "--exclude-compressed:  only compress files that are not previously compressed \n");
     DISPLAY( " -f     : overwrite output without prompting and (de)compress links \n");
     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
     DISPLAY( " -k     : preserve source file(s) (default) \n");
@@ -708,7 +709,7 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
                     if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
                     if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
-
+                    if (!strcmp(argument, "--exclude-compressed")) { g_excludeCompressedFiles = 1; continue; }
                     /* long commands with arguments */
 #ifndef ZSTD_NODICT
                     if (longCommandWArg(&argument, "--train-cover")) {
diff --git a/tests/playTests.sh b/tests/playTests.sh
index f68ee81a5..0946c0326 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -215,6 +215,25 @@ $ZSTD tmp -c --compress-literals    -19      | $ZSTD -t
 $ZSTD -b --fast=1 -i0e1 tmp --compress-literals
 $ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
 
+println "test: --exclude-compressed flag"
+mkdir precompressedFilterTestDir
+./datagen $size > precompressedFilterTestDir/input.5
+./datagen $size > precompressedFilterTestDir/input.6
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+sleep 5
+./datagen $size > precompressedFilterTestDir/input.7
+./datagen $size > precompressedFilterTestDir/input.8
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
+file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
+if [[ $file2timestamp -ge $file1timestamp ]]; then
+  println "Test is successful. input.5.zst is not precompressed and therefore not compressed/modified again."
+else
+  println "Test is not successful"
+fi
+println "Test completed"
+sleep 5
+
 println "test : file removal"
 $ZSTD -f --rm tmp
 test ! -f tmp  # tmp should no longer be present

From 48f856640e802d7f15990d2973f3fb7562f18ce5 Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Fri, 25 Oct 2019 15:49:11 -0700
Subject: [PATCH 06/27] Added --exclude-compressed flag feature that skips
 compression of precompressed files

---
 programs/fileio.c  |  7 ++++++-
 programs/util.c    | 21 +++++++++++++++++++++
 programs/util.h    |  4 ++++
 programs/zstdcli.c |  3 ++-
 tests/playTests.sh | 19 +++++++++++++++++++
 5 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 828878c6a..ff3401b5d 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1453,7 +1453,12 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
 
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
-
+    if (g_excludeCompressedFiles && !UTIL_isPrecompressedFile(srcFileName)) {  /* precompressed file (--exclude-compressed). DO NOT COMPRESS */
+        DISPLAYLEVEL(4, "Precompressed file: %s \n", srcFileName);
+        fclose(ress.srcFile);
+        ress.srcFile = NULL;
+        return 0;
+    }
     result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);
diff --git a/programs/util.c b/programs/util.c
index 587058805..830f2039e 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -326,6 +326,27 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
 
 #endif /* #ifdef _WIN32 */
 
+/* Check if the file is precompressed (.zst, .lz4, .gz, .xz).
+YES => Skip the file (return 0)
+NO => return 1
+*/
+int UTIL_isPrecompressedFile(const char *inputName)
+{
+    return compareExtensions(inputName,compressedFileExtensions);
+}
+
+int compareExtensions(const char* infilename, const char extensionList[4][10])
+{
+   int i=0;
+   //char* ext = strchr(infilename, '.');
+   for(i=0;i<4;i++)
+   {
+     char* ext = strstr(infilename,extensionList[i]);
+     if(ext)
+        return 0;
+   }
+   return 1;
+}
 /*
  * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
  *                       and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
diff --git a/programs/util.h b/programs/util.h
index 71ba0d4fc..0da6255c0 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -127,6 +127,8 @@ extern int g_utilDisplayLevel;
     typedef struct stat stat_t;
 #endif
 
+int g_excludeCompressedFiles;
+static const char compressedFileExtensions[4][10] = {".zst",".gz",".xz",".lz4"};
 
 int UTIL_fileExist(const char* filename);
 int UTIL_isRegularFile(const char* infilename);
@@ -135,6 +137,8 @@ U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 int UTIL_compareStr(const void *p1, const void *p2);
+int UTIL_isPrecompressedFile(const char* infilename);
+int compareExtensions(const char* infilename, const char extensionList[4][10]);
 
 U32 UTIL_isFIFO(const char* infilename);
 U32 UTIL_isLink(const char* infilename);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 3d5c4280e..11116afc0 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -118,6 +118,7 @@ static int usage(const char* programName)
 #endif
     DISPLAY( " -D file: use `file` as Dictionary \n");
     DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
+    DISPLAY( "--exclude-compressed:  only compress files that are not previously compressed \n");
     DISPLAY( " -f     : overwrite output without prompting and (de)compress links \n");
     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
     DISPLAY( " -k     : preserve source file(s) (default) \n");
@@ -708,7 +709,7 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
                     if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
                     if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
-
+                    if (!strcmp(argument, "--exclude-compressed")) { g_excludeCompressedFiles = 1; continue; }
                     /* long commands with arguments */
 #ifndef ZSTD_NODICT
                     if (longCommandWArg(&argument, "--train-cover")) {
diff --git a/tests/playTests.sh b/tests/playTests.sh
index c1da16507..9294bf812 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -215,6 +215,25 @@ $ZSTD tmp -c --compress-literals    -19      | $ZSTD -t
 $ZSTD -b --fast=1 -i0e1 tmp --compress-literals
 $ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
 
+println "test: --exclude-compressed flag"
+mkdir precompressedFilterTestDir
+./datagen $size > precompressedFilterTestDir/input.5
+./datagen $size > precompressedFilterTestDir/input.6
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+sleep 5
+./datagen $size > precompressedFilterTestDir/input.7
+./datagen $size > precompressedFilterTestDir/input.8
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
+file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
+if [[ $file2timestamp -ge $file1timestamp ]]; then
+  println "Test is successful. input.5.zst is not precompressed and therefore not compressed/modified again."
+else
+  println "Test is not successful"
+fi
+println "Test completed"
+sleep 5
+
 println "test : file removal"
 $ZSTD -f --rm tmp
 test ! -f tmp  # tmp should no longer be present

From 02433e0b1556c8c9d769471226b12209dd6512c8 Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Mon, 28 Oct 2019 14:54:54 -0700
Subject: [PATCH 07/27] Addressing comments: -Created a list of extensions
 defined in fileio.h, -Updated test

---
 programs/fileio.c  |  9 +++++++--
 programs/util.c    | 17 ++++++++---------
 programs/util.h    | 19 +++++++++++++++----
 programs/zstdcli.c |  2 +-
 tests/playTests.sh |  1 -
 5 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index ff3401b5d..7cbf0280e 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1453,8 +1453,13 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
 
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
-    if (g_excludeCompressedFiles && !UTIL_isPrecompressedFile(srcFileName)) {  /* precompressed file (--exclude-compressed). DO NOT COMPRESS */
-        DISPLAYLEVEL(4, "Precompressed file: %s \n", srcFileName);
+
+    /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
+    * YES => ZSTD will not compress the file.
+    * NO => ZSTD will resume with compress operation.
+    */
+    if (g_excludeCompressedFiles && UTIL_isCompressedFile(srcFileName)) {  /* precompressed file (--exclude-compressed). DO NOT COMPRESS */
+        DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
         fclose(ress.srcFile);
         ress.srcFile = NULL;
         return 0;
diff --git a/programs/util.c b/programs/util.c
index 830f2039e..63e9ef9e9 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -330,22 +330,21 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
 YES => Skip the file (return 0)
 NO => return 1
 */
-int UTIL_isPrecompressedFile(const char *inputName)
+int UTIL_isCompressedFile(const char *inputName)
 {
-    return compareExtensions(inputName,compressedFileExtensions);
+    return compareExtensions(inputName,g_compressedFileExtensions);
 }
 
-int compareExtensions(const char* infilename, const char extensionList[4][10])
+int compareExtensions(const char* infilename, const char* extensionList[])
 {
-   int i=0;
-   //char* ext = strchr(infilename, '.');
-   for(i=0;i<4;i++)
+   while(*extensionList != NULL)
    {
-     char* ext = strstr(infilename,extensionList[i]);
+     const char* ext = strstr(infilename,extensionList[i]);
      if(ext)
-        return 0;
+        return 1;
+      ++extensionList;
    }
-   return 1;
+   return 0;
 }
 /*
  * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
diff --git a/programs/util.h b/programs/util.h
index 0da6255c0..deb70786a 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -39,7 +39,7 @@ extern "C" {
 #endif
 #include <time.h>         /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
 #include "mem.h"          /* U32, U64 */
-
+#include "fileio.h"
 
 /*-************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
@@ -128,7 +128,18 @@ extern int g_utilDisplayLevel;
 #endif
 
 int g_excludeCompressedFiles;
-static const char compressedFileExtensions[4][10] = {".zst",".gz",".xz",".lz4"};
+static const char *g_compressedFileExtensions[] = {
+    ZSTD_EXTENSION,
+    TZSTD_EXTENSION,
+    GZ_EXTENSION,
+    TGZ_EXTENSION,
+    LZMA_EXTENSION,
+    XZ_EXTENSION,
+    TXZ_EXTENSION,
+    LZ4_EXTENSION,
+    TLZ4_EXTENSION,
+    NULL
+};
 
 int UTIL_fileExist(const char* filename);
 int UTIL_isRegularFile(const char* infilename);
@@ -137,8 +148,8 @@ U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 int UTIL_compareStr(const void *p1, const void *p2);
-int UTIL_isPrecompressedFile(const char* infilename);
-int compareExtensions(const char* infilename, const char extensionList[4][10]);
+int UTIL_isCompressedFile(const char* infilename);
+int compareExtensions(const char* infilename, const char *extensionList[]);
 
 U32 UTIL_isFIFO(const char* infilename);
 U32 UTIL_isLink(const char* infilename);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 11116afc0..a704a1abd 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -118,7 +118,6 @@ static int usage(const char* programName)
 #endif
     DISPLAY( " -D file: use `file` as Dictionary \n");
     DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
-    DISPLAY( "--exclude-compressed:  only compress files that are not previously compressed \n");
     DISPLAY( " -f     : overwrite output without prompting and (de)compress links \n");
     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
     DISPLAY( " -k     : preserve source file(s) (default) \n");
@@ -137,6 +136,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
     DISPLAY( " -l     : print information about zstd compressed files \n");
+    DISPLAY( "--exclude-compressed:  only compress files that are not previously compressed \n");
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
     DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 9294bf812..ca286071d 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -232,7 +232,6 @@ else
   println "Test is not successful"
 fi
 println "Test completed"
-sleep 5
 
 println "test : file removal"
 $ZSTD -f --rm tmp

From 0e9a37daeb4a9ac0153ccf7cc144732fe5ee42bc Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Mon, 28 Oct 2019 15:22:26 -0700
Subject: [PATCH 08/27] Fixing tests and moving flag to advanced usage

---
 programs/zstdcli.c | 1 -
 tests/playTests.sh | 1 -
 2 files changed, 2 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index a5a3f30da..a704a1abd 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -118,7 +118,6 @@ static int usage(const char* programName)
 #endif
     DISPLAY( " -D file: use `file` as Dictionary \n");
     DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
-    DISPLAY( "--exclude-compressed:  only compress files that are not previously compressed \n");
     DISPLAY( " -f     : overwrite output without prompting and (de)compress links \n");
     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
     DISPLAY( " -k     : preserve source file(s) (default) \n");
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 9294bf812..ca286071d 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -232,7 +232,6 @@ else
   println "Test is not successful"
 fi
 println "Test completed"
-sleep 5
 
 println "test : file removal"
 $ZSTD -f --rm tmp

From 9c1860861ef15891f60510f9d4c3d6a75da2e12f Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 28 Oct 2019 17:51:17 -0700
Subject: [PATCH 09/27] Fix assert in ZSTD_safecopy

In the case that `op >= oend_w` it is possible that `diff < 8` because
the two buffers could be adjacent.

Credit to OSS-Fuzz, which found the bug. It isn't reproducible because
it depends on the memory layout.
---
 lib/decompress/zstd_decompress_block.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index cbb66c8db..767e5f9a0 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -617,7 +617,7 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_
     ptrdiff_t const diff = op - ip;
     BYTE* const oend = op + length;
 
-    assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8)) ||
+    assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) ||
            (ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
 
     if (length < 8) {

From 0f2bff2faf0cb60c4fe5346f56a5b42558a8e1a9 Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Mon, 28 Oct 2019 18:21:47 -0700
Subject: [PATCH 10/27] Addressing comments, removing cyclic dependency with
 header file, updating tests

---
 programs/fileio.c  | 36 ++++++++++++++++++++++++++----------
 programs/fileio.h  |  1 +
 programs/util.c    | 14 ++++----------
 programs/util.h    | 18 +-----------------
 programs/zstdcli.c |  2 +-
 tests/playTests.sh |  3 +++
 6 files changed, 36 insertions(+), 38 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 7cbf0280e..e59fb80f6 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -319,6 +319,8 @@ struct FIO_prefs_s {
     /* Computation resources preferences */
     unsigned memLimit;
     int nbWorkers;
+
+    int excludeCompressedFiles;
 };
 
 
@@ -359,6 +361,7 @@ FIO_prefs_t* FIO_createPreferences(void)
     ret->srcSizeHint = 0;
     ret->testMode = 0;
     ret->literalCompressionMode = ZSTD_lcm_auto;
+    ret->excludeCompressedFiles = 0;
     return ret;
 }
 
@@ -402,6 +405,8 @@ void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) {
     prefs->nbWorkers = nbWorkers;
 }
 
+void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; }
+
 void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
     if (blockSize && prefs->nbWorkers==0)
         DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
@@ -1425,6 +1430,18 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
     return result;
 }
 
+static const char *compressedFileExtensions[] = {
+    ZSTD_EXTENSION,
+    TZSTD_EXTENSION,
+    GZ_EXTENSION,
+    TGZ_EXTENSION,
+    LZMA_EXTENSION,
+    XZ_EXTENSION,
+    TXZ_EXTENSION,
+    LZ4_EXTENSION,
+    TLZ4_EXTENSION,
+    NULL
+};
 
 /*! FIO_compressFilename_srcFile() :
  *  @return : 0 : compression completed correctly,
@@ -1451,19 +1468,18 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
         return 1;
     }
 
+    /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
+    * YES => ZSTD will skip compression of the file and will return 0.
+    * NO => ZSTD will resume with compress operation.
+    */
+    if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) {
+        DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
+        return 0;
+    }
+
     ress.srcFile = FIO_openSrcFile(srcFileName);
     if (ress.srcFile == NULL) return 1;   /* srcFile could not be opened */
 
-    /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
-    * YES => ZSTD will not compress the file.
-    * NO => ZSTD will resume with compress operation.
-    */
-    if (g_excludeCompressedFiles && UTIL_isCompressedFile(srcFileName)) {  /* precompressed file (--exclude-compressed). DO NOT COMPRESS */
-        DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
-        fclose(ress.srcFile);
-        ress.srcFile = NULL;
-        return 0;
-    }
     result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);
diff --git a/programs/fileio.h b/programs/fileio.h
index af2c5d9d1..a7da089f6 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -93,6 +93,7 @@ void FIO_setLiteralCompressionMode(
 
 void FIO_setNoProgress(unsigned noProgress);
 void FIO_setNotificationLevel(int level);
+void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles);
 
 /*-*************************************
 *  Single File functions
diff --git a/programs/util.c b/programs/util.c
index f82079554..7212f7b81 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -331,24 +331,18 @@ YES => Skip the file (return 0)
 NO => return 1
 */
 
-int UTIL_isCompressedFile(const char *inputName)
+int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
 {
-    return compareExtensions(inputName,g_compressedFileExtensions);
-}
-
-int compareExtensions(const char* infilename, const char* extensionList[])
-{
-  int i=0;
-   while(*extensionList != NULL)
+   while(*extensionList!=NULL)
    {
-     const char* ext = strstr(infilename,extensionList[i]);
+     const char* ext = strstr(inputName,*extensionList);
      if(ext)
         return 1;
       ++extensionList;
-      i++;
    }
    return 0;
 }
+
 /*
  * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
  *                       and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
diff --git a/programs/util.h b/programs/util.h
index deb70786a..deaea0323 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -39,7 +39,6 @@ extern "C" {
 #endif
 #include <time.h>         /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
 #include "mem.h"          /* U32, U64 */
-#include "fileio.h"
 
 /*-************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
@@ -127,19 +126,6 @@ extern int g_utilDisplayLevel;
     typedef struct stat stat_t;
 #endif
 
-int g_excludeCompressedFiles;
-static const char *g_compressedFileExtensions[] = {
-    ZSTD_EXTENSION,
-    TZSTD_EXTENSION,
-    GZ_EXTENSION,
-    TGZ_EXTENSION,
-    LZMA_EXTENSION,
-    XZ_EXTENSION,
-    TXZ_EXTENSION,
-    LZ4_EXTENSION,
-    TLZ4_EXTENSION,
-    NULL
-};
 
 int UTIL_fileExist(const char* filename);
 int UTIL_isRegularFile(const char* infilename);
@@ -148,9 +134,7 @@ U32 UTIL_isDirectory(const char* infilename);
 int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 int UTIL_compareStr(const void *p1, const void *p2);
-int UTIL_isCompressedFile(const char* infilename);
-int compareExtensions(const char* infilename, const char *extensionList[]);
-
+int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
 U32 UTIL_isFIFO(const char* infilename);
 U32 UTIL_isLink(const char* infilename);
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index a704a1abd..5463c019f 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -709,7 +709,7 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
                     if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
                     if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
-                    if (!strcmp(argument, "--exclude-compressed")) { g_excludeCompressedFiles = 1; continue; }
+                    if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
                     /* long commands with arguments */
 #ifndef ZSTD_NODICT
                     if (longCommandWArg(&argument, "--train-cover")) {
diff --git a/tests/playTests.sh b/tests/playTests.sh
index ca286071d..cb703227b 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -224,6 +224,8 @@ sleep 5
 ./datagen $size > precompressedFilterTestDir/input.7
 ./datagen $size > precompressedFilterTestDir/input.8
 $ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+test ! -f input.5.zst.zst
+test ! -f input.6.zst.zst
 file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
 file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
 if [[ $file2timestamp -ge $file1timestamp ]]; then
@@ -232,6 +234,7 @@ else
   println "Test is not successful"
 fi
 println "Test completed"
+sleep 5
 
 println "test : file removal"
 $ZSTD -f --rm tmp

From 66f580ca73451dd8114a0c7ab43944ec7b6fe30e Mon Sep 17 00:00:00 2001
From: Bimba Shrestha <bimbashrestha@fb.com>
Date: Mon, 28 Oct 2019 22:09:34 -0700
Subject: [PATCH 11/27] Removing Visual08 and Visual10 tests

---
 appveyor.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 3a4803363..9958c882d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -173,8 +173,6 @@
       sh -e playTests.sh --test-large-data &&
       fullbench.exe -i1 &&
       fullbench.exe -i1 -P0 &&
-      fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
-      fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
       fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
       fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
       fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%

From 850ba66139001f2caee0694efbc0e2e53bbbfc5f Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Mon, 28 Oct 2019 22:24:01 -0700
Subject: [PATCH 12/27] Minor fixes in test

---
 tests/playTests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index cb703227b..23931a4aa 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -216,7 +216,7 @@ $ZSTD -b --fast=1 -i0e1 tmp --compress-literals
 $ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
 
 println "test: --exclude-compressed flag"
-mkdir precompressedFilterTestDir
+mkdir -p precompressedFilterTestDir
 ./datagen $size > precompressedFilterTestDir/input.5
 ./datagen $size > precompressedFilterTestDir/input.6
 $ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir

From 26ffc1863483d3cb4c24e2e7638a301324693c49 Mon Sep 17 00:00:00 2001
From: 0x123456789A <57141050+0x123456789A@users.noreply.github.com>
Date: Tue, 29 Oct 2019 14:31:02 +0100
Subject: [PATCH 13/27] Introduce ZSTD_PROGRAMS_LINK_SHARED

The CMake variable ZSTD_PROGRAMS_LINK_SHARED indicactes wether or not to link the zstd programs dynamically or statically.
---
 build/cmake/CMakeLists.txt | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index 33c05aee8..41fdf787f 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -124,9 +124,13 @@ endif ()
 #-----------------------------------------------------------------------------
 add_subdirectory(lib)
 
+option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
+
 if (ZSTD_BUILD_PROGRAMS)
-    if (NOT ZSTD_BUILD_STATIC)
+    if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
         message(SEND_ERROR "You need to build static library to build zstd CLI")
+    elseif(ZSTD_BUILD_STATIC AND ZSTD_PROGRAMS_LINK_SHARED)
+        message(SEND_ERROR "You need to build shared library to build zstd CLI")
     endif ()
 
     add_subdirectory(programs)

From 57a311d3b7dac3728a366ef9892f9d0ed48816a8 Mon Sep 17 00:00:00 2001
From: 0x123456789A <57141050+0x123456789A@users.noreply.github.com>
Date: Tue, 29 Oct 2019 14:33:50 +0100
Subject: [PATCH 14/27] Consider ZSTD_PROGRAMS_LINK_SHARED

Actually consider ZSTD_PROGRAMS_LINK_SHARED in programs CMakeLists
---
 build/cmake/programs/CMakeLists.txt | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index 8afd83503..b26e97d0d 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -21,13 +21,19 @@ if (ZSTD_LEGACY_SUPPORT)
     include_directories(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
 endif ()
 
+if (ZSTD_PROGRAMS_LINK_SHARED)
+    set(PROGRAMS_ZSTD_LINK_TARGET libzstd_shared)
+else ()
+    set(PROGRAMS_ZSTD_LINK_TARGET libzstd_static)
+endif ()
+
 if (MSVC)
     set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/zstd)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
 endif ()
 
 add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
-target_link_libraries(zstd libzstd_static)
+target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
 if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
     target_link_libraries(zstd rt)
 endif ()
@@ -68,7 +74,7 @@ if (UNIX)
         DESTINATION "${MAN_INSTALL_DIR}")
 
     add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
-    target_link_libraries(zstd-frugal libzstd_static)
+    target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
     set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
 endif ()
 

From 9db11e3e42d9cde6bfd88e0c52df369aefe55d5a Mon Sep 17 00:00:00 2001
From: 0x123456789A <57141050+0x123456789A@users.noreply.github.com>
Date: Tue, 29 Oct 2019 14:41:32 +0100
Subject: [PATCH 15/27] Fixed check for building programs statically

---
 build/cmake/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index 41fdf787f..9d0e7fb0b 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -129,7 +129,7 @@ option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
 if (ZSTD_BUILD_PROGRAMS)
     if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
         message(SEND_ERROR "You need to build static library to build zstd CLI")
-    elseif(ZSTD_BUILD_STATIC AND ZSTD_PROGRAMS_LINK_SHARED)
+    elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
         message(SEND_ERROR "You need to build shared library to build zstd CLI")
     endif ()
 

From 4a9eca4b9d40631e87b37a8e31a6a410f3bd6af0 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha <bimbashrestha@fb.com>
Date: Tue, 29 Oct 2019 09:45:28 -0700
Subject: [PATCH 16/27] Removing merge side effect

---
 appveyor.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 9958c882d..dd2c02ac4 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -262,6 +262,5 @@
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
   - if [%HOST%]==[mingw] (
       set "CC=%COMPILER%" &&
-      make check &&
-      sh -e playTests.sh --test-large-data
+      make check
     )

From 5e6dbad6c1c038d34c2be67edc500a012e9efb1a Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Tue, 29 Oct 2019 09:54:54 -0700
Subject: [PATCH 17/27] Deleting test directory before creation to pass
 travis-ci test

---
 tests/playTests.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 23931a4aa..d36a9d91c 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -216,6 +216,7 @@ $ZSTD -b --fast=1 -i0e1 tmp --compress-literals
 $ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
 
 println "test: --exclude-compressed flag"
+rm -rf precompressedFilterTestDir
 mkdir -p precompressedFilterTestDir
 ./datagen $size > precompressedFilterTestDir/input.5
 ./datagen $size > precompressedFilterTestDir/input.6

From 9ab6a747d40144c804d8a23000c58389931423ff Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Tue, 29 Oct 2019 12:27:54 -0700
Subject: [PATCH 18/27] Created utility function to extract extension from
 filename, fixed tests

---
 programs/fileio.c  |  3 +++
 programs/util.c    | 19 ++++++++++++++-----
 programs/util.h    |  2 ++
 tests/playTests.sh |  3 +--
 4 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index e59fb80f6..c0d6494ec 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -1430,6 +1430,9 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
     return result;
 }
 
+/* List used to compare file extensions (used with --exclude-compressed flag)
+* Different from the suffixList and should only apply to ZSTD compress operationResult
+*/
 static const char *compressedFileExtensions[] = {
     ZSTD_EXTENSION,
     TZSTD_EXTENSION,
diff --git a/programs/util.c b/programs/util.c
index 7212f7b81..f6933bc83 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -326,23 +326,32 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
 
 #endif /* #ifdef _WIN32 */
 
-/* Check if the file is precompressed (.zst, .lz4, .gz, .xz).
-YES => Skip the file (return 0)
-NO => return 1
+/* Check if the file is Compressed by comparing it with compressFileExtension list.
+YES => Skip the file (return 1)
+NO => return 0
 */
 
 int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
 {
+  const char* ext = UTIL_getFileExtension(inputName);
    while(*extensionList!=NULL)
    {
-     const char* ext = strstr(inputName,*extensionList);
-     if(ext)
+     const char* isCompressedExtension = strstr(ext,*extensionList);
+     if(isCompressedExtension)
         return 1;
       ++extensionList;
    }
    return 0;
 }
 
+/*Utility function to get file extension from file */
+const char* UTIL_getFileExtension(const char* infilename)
+{
+   const char* extension = strrchr(infilename, '.');
+   if(!extension || extension==infilename) return "";
+   return extension;
+}
+
 /*
  * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
  *                       and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
diff --git a/programs/util.h b/programs/util.h
index deaea0323..0d0642cb5 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -135,6 +135,8 @@ int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
 int UTIL_isSameFile(const char* file1, const char* file2);
 int UTIL_compareStr(const void *p1, const void *p2);
 int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
+const char* UTIL_getFileExtension(const char* infilename);
+
 U32 UTIL_isFIFO(const char* infilename);
 U32 UTIL_isLink(const char* infilename);
 #define UTIL_FILESIZE_UNKNOWN  ((U64)(-1))
diff --git a/tests/playTests.sh b/tests/playTests.sh
index d36a9d91c..6161983ef 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -230,12 +230,11 @@ test ! -f input.6.zst.zst
 file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
 file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
 if [[ $file2timestamp -ge $file1timestamp ]]; then
-  println "Test is successful. input.5.zst is not precompressed and therefore not compressed/modified again."
+  println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again."
 else
   println "Test is not successful"
 fi
 println "Test completed"
-sleep 5
 
 println "test : file removal"
 $ZSTD -f --rm tmp

From c5060997e9c8ccc57e2fcc3c8bc0246cb3b93749 Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Tue, 29 Oct 2019 12:56:04 -0700
Subject: [PATCH 19/27] Added check to perform comparison only if extension is
 present

---
 programs/util.c | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index f6933bc83..c93382c63 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -334,13 +334,16 @@ NO => return 0
 int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
 {
   const char* ext = UTIL_getFileExtension(inputName);
-   while(*extensionList!=NULL)
-   {
-     const char* isCompressedExtension = strstr(ext,*extensionList);
-     if(isCompressedExtension)
-        return 1;
-      ++extensionList;
-   }
+  if(strcmp(ext,""))
+  {
+     while(*extensionList!=NULL)
+     {
+       const char* isCompressedExtension = strstr(ext,*extensionList);
+       if(isCompressedExtension)
+          return 1;
+        ++extensionList;
+     }
+  }
    return 0;
 }
 

From 6d0b7bd6ce797c27f6c977f645f946c97dda4d1a Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Tue, 29 Oct 2019 13:36:09 -0700
Subject: [PATCH 20/27] Changed extension comparision logic, added new test
 cases

---
 programs/util.c    |  9 ++-------
 tests/playTests.sh | 14 ++++++++++++--
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index c93382c63..9afb5a869 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -326,11 +326,6 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
 
 #endif /* #ifdef _WIN32 */
 
-/* Check if the file is Compressed by comparing it with compressFileExtension list.
-YES => Skip the file (return 1)
-NO => return 0
-*/
-
 int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
 {
   const char* ext = UTIL_getFileExtension(inputName);
@@ -338,8 +333,8 @@ int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
   {
      while(*extensionList!=NULL)
      {
-       const char* isCompressedExtension = strstr(ext,*extensionList);
-       if(isCompressedExtension)
+       const int isCompressedExtension = strcmp(ext,*extensionList);
+       if(isCompressedExtension==0)
           return 1;
         ++extensionList;
      }
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 6161983ef..4672ecaa3 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -225,8 +225,8 @@ sleep 5
 ./datagen $size > precompressedFilterTestDir/input.7
 ./datagen $size > precompressedFilterTestDir/input.8
 $ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
-test ! -f input.5.zst.zst
-test ! -f input.6.zst.zst
+test ! -f precompressedFilterTestDir/input.5.zst.zst
+test ! -f precompressedFilterTestDir/input.6.zst.zst
 file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
 file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
 if [[ $file2timestamp -ge $file1timestamp ]]; then
@@ -234,6 +234,16 @@ if [[ $file2timestamp -ge $file1timestamp ]]; then
 else
   println "Test is not successful"
 fi
+#File Extension check.
+./datagen $size > precompressedFilterTestDir/input.zstbar
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+#ZSTD should compress input.zstbar
+test -f precompressedFilterTestDir/input.zstbar.zst
+#Check without the --exclude-compressed flag
+$ZSTD --long --rm -r precompressedFilterTestDir
+#Files should get compressed again without the --exclude-compressed flag.
+test -f precompressedFilterTestDir/input.5.zst.zst
+test -f precompressedFilterTestDir/input.6.zst.zst
 println "Test completed"
 
 println "test : file removal"

From 3c1649f1395b93a55f66cb307989c556cc5418f8 Mon Sep 17 00:00:00 2001
From: Shashank Tavildar <shtavi@fb.com>
Date: Tue, 29 Oct 2019 15:59:20 -0700
Subject: [PATCH 21/27] Removed the optimization check

---
 programs/util.c | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index 9afb5a869..2143d178b 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -329,15 +329,12 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
 int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
 {
   const char* ext = UTIL_getFileExtension(inputName);
-  if(strcmp(ext,""))
+  while(*extensionList!=NULL)
   {
-     while(*extensionList!=NULL)
-     {
-       const int isCompressedExtension = strcmp(ext,*extensionList);
-       if(isCompressedExtension==0)
-          return 1;
-        ++extensionList;
-     }
+    const int isCompressedExtension = strcmp(ext,*extensionList);
+    if(isCompressedExtension==0)
+      return 1;
+    ++extensionList;
   }
    return 0;
 }

From ae3d35dc8e40368aff29c8b431b5555a9c3683b5 Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Wed, 30 Oct 2019 12:16:45 -0700
Subject: [PATCH 22/27] updated CHANGELOG for v1.4.4

---
 CHANGELOG | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG b/CHANGELOG
index 44600267d..3b882d4cd 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -11,6 +11,7 @@ cli: Named pipes support, by @bimbashrestha
 cli: short tar's extension support, by @stokito
 cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42
 cli: commands --stream-size=# and --size-hint=#, by @nmagerko
+cli: command --exclude-compressed, by @shashank0791
 cli: faster `-t` test mode
 cli: improved some error messages, by @vangyzen
 cli: rare deadlock condition within dictionary builder, by @terrelln

From b9ede1c8c2accdb3593c5154ae48d2edace7c1b3 Mon Sep 17 00:00:00 2001
From: Sen Huang <senhuang96@fb.com>
Date: Wed, 30 Oct 2019 16:03:58 -0400
Subject: [PATCH 23/27] Make sure contentsize is known

---
 lib/compress/zstd_compress.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 07df9f04b..a8856cd1b 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2934,6 +2934,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
       && (cdict->dictContentSize > 0)
       && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
         || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
+        || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
         || cdict->compressionLevel == 0)
       && (params->attachDictPref != ZSTD_dictForceLoad) ) {
         return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
@@ -3381,6 +3382,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     {   ZSTD_CCtx_params params = cctx->requestedParams;
         params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
                         || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
+                        || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
                         || cdict->compressionLevel == 0 )
                       && (params.attachDictPref != ZSTD_dictForceLoad) ?
                 ZSTD_getCParamsFromCDict(cdict)

From d770a2a89ff9435e3283c88ad237f9a70f872c9a Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Thu, 31 Oct 2019 10:47:06 -0700
Subject: [PATCH 24/27] fixed zlibwrapper build script

---
 zlibWrapper/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index f63291c90..6addb743c 100644
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -103,6 +103,7 @@ zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/z
 
 zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1
 zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< -c -o $@
 
 $(ZSTDLIBDIR)/libzstd.a:
 	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a

From 75e7c0d107e17b6ce7392d82aa8e9122a826d6f2 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 1 Nov 2019 15:16:24 -0700
Subject: [PATCH 25/27] [fuzz] Add dictionary_loader fuzzer

* Adds the fuzzer
* Adds an additional `InputType` for the fuzzer

I ran the fuzzer for about 10 minutes and it found 2 bugs:

* Catches the original bug without any help
* Catches an additional bug with 8-byte dictionaries
---
 tests/fuzz/Makefile            |  6 ++-
 tests/fuzz/dictionary_loader.c | 93 ++++++++++++++++++++++++++++++++
 tests/fuzz/fuzz.py             | 98 ++++++++++++++++++++++------------
 3 files changed, 161 insertions(+), 36 deletions(-)
 create mode 100644 tests/fuzz/dictionary_loader.c

diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 83837e620..41b634cab 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -73,7 +73,8 @@ FUZZ_TARGETS :=       \
 	dictionary_round_trip \
 	dictionary_decompress \
 	zstd_frame_info \
-	simple_compress
+	simple_compress \
+	dict_loader
 
 all: $(FUZZ_TARGETS)
 
@@ -110,6 +111,9 @@ simple_compress: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_compress.o
 zstd_frame_info: $(FUZZ_HEADERS) $(FUZZ_OBJ) zstd_frame_info.o
 	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) zstd_frame_info.o $(LIB_FUZZING_ENGINE) -o $@
 
+dictionary_loader: $(FUZZ_HEADERS) $(FUZZ_OBJ) dictionary_loader.o
+	$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) dictionary_loader.o $(LIB_FUZZING_ENGINE) -o $@
+
 libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c regression_driver.o
 	$(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o
 
diff --git a/tests/fuzz/dictionary_loader.c b/tests/fuzz/dictionary_loader.c
new file mode 100644
index 000000000..cb34f5d22
--- /dev/null
+++ b/tests/fuzz/dictionary_loader.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+/**
+ * This fuzz target makes sure that whenever a compression dictionary can be
+ * loaded, the data can be round tripped.
+ */
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "fuzz_helpers.h"
+#include "zstd_helpers.h"
+#include "fuzz_data_producer.h"
+
+/**
+ * Compresses the data and returns the compressed size or an error.
+ */
+static size_t compress(void* compressed, size_t compressedCapacity,
+                       void const* source, size_t sourceSize,
+                       void const* dict, size_t dictSize,
+                       ZSTD_dictLoadMethod_e dictLoadMethod,
+                       ZSTD_dictContentType_e dictContentType)
+{
+    ZSTD_CCtx* cctx = ZSTD_createCCtx();
+    FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
+            cctx, dict, dictSize, dictLoadMethod, dictContentType));
+    size_t const compressedSize = ZSTD_compress2(
+            cctx, compressed, compressedCapacity, source, sourceSize);
+    ZSTD_freeCCtx(cctx);
+    return compressedSize;
+}
+
+static size_t decompress(void* result, size_t resultCapacity,
+                         void const* compressed, size_t compressedSize,
+                         void const* dict, size_t dictSize,
+                       ZSTD_dictLoadMethod_e dictLoadMethod,
+                         ZSTD_dictContentType_e dictContentType)
+{
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
+    FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
+            dctx, dict, dictSize, dictLoadMethod, dictContentType));
+    size_t const resultSize = ZSTD_decompressDCtx(
+            dctx, result, resultCapacity, compressed, compressedSize);
+    FUZZ_ZASSERT(resultSize);
+    ZSTD_freeDCtx(dctx);
+    return resultSize;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+{
+    FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+    ZSTD_dictLoadMethod_e const dlm =
+    size = FUZZ_dataProducer_uint32Range(producer, 0, 1);
+    ZSTD_dictContentType_e const dct =
+            FUZZ_dataProducer_uint32Range(producer, 0, 2);
+    size = FUZZ_dataProducer_remainingBytes(producer);
+
+    DEBUGLOG(2, "Dict load method %d", dlm);
+    DEBUGLOG(2, "Dict content type %d", dct);
+    DEBUGLOG(2, "Dict size %u", (unsigned)size);
+
+    void* const rBuf = malloc(size);
+    FUZZ_ASSERT(rBuf);
+    size_t const cBufSize = ZSTD_compressBound(size);
+    void* const cBuf = malloc(cBufSize);
+    FUZZ_ASSERT(cBuf);
+
+    size_t const cSize =
+            compress(cBuf, cBufSize, src, size, src, size, dlm, dct);
+    /* compression failing is okay */
+    if (ZSTD_isError(cSize)) {
+      FUZZ_ASSERT_MSG(dct != ZSTD_dct_rawContent, "Raw must always succeed!");
+      goto out;
+    }
+    size_t const rSize =
+            decompress(rBuf, size, cBuf, cSize, src, size, dlm, dct);
+    FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
+    FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
+
+out:
+    free(cBuf);
+    free(rBuf);
+    FUZZ_dataProducer_free(producer);
+    return 0;
+}
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index 9df68df0c..87f115afd 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -27,6 +27,7 @@ def abs_join(a, *p):
 class InputType(object):
     RAW_DATA = 1
     COMPRESSED_DATA = 2
+    DICTIONARY_DATA = 3
 
 
 class FrameType(object):
@@ -54,6 +55,7 @@ TARGET_INFO = {
     'dictionary_decompress': TargetInfo(InputType.COMPRESSED_DATA),
     'zstd_frame_info': TargetInfo(InputType.COMPRESSED_DATA),
     'simple_compress': TargetInfo(InputType.RAW_DATA),
+    'dictionary_loader': TargetInfo(InputType.DICTIONARY_DATA),
 }
 TARGETS = list(TARGET_INFO.keys())
 ALL_TARGETS = TARGETS + ['all']
@@ -73,6 +75,7 @@ LIB_FUZZING_ENGINE = os.environ.get('LIB_FUZZING_ENGINE', 'libregression.a')
 AFL_FUZZ = os.environ.get('AFL_FUZZ', 'afl-fuzz')
 DECODECORPUS = os.environ.get('DECODECORPUS',
                               abs_join(FUZZ_DIR, '..', 'decodecorpus'))
+ZSTD = os.environ.get('ZSTD', abs_join(FUZZ_DIR, '..', '..', 'zstd'))
 
 # Sanitizer environment variables
 MSAN_EXTRA_CPPFLAGS = os.environ.get('MSAN_EXTRA_CPPFLAGS', '')
@@ -673,6 +676,11 @@ def gen_parser(args):
         default=DECODECORPUS,
         help="decodecorpus binary (default: $DECODECORPUS='{}')".format(
             DECODECORPUS))
+    parser.add_argument(
+        '--zstd',
+        type=str,
+        default=ZSTD,
+        help="zstd binary (default: $ZSTD='{}')".format(ZSTD))
     parser.add_argument(
         '--fuzz-rng-seed-size',
         type=int,
@@ -707,46 +715,66 @@ def gen(args):
         return 1
 
     seed = create(args.seed)
-    with tmpdir() as compressed:
-        with tmpdir() as decompressed:
-            cmd = [
-                args.decodecorpus,
-                '-n{}'.format(args.number),
-                '-p{}/'.format(compressed),
-                '-o{}'.format(decompressed),
+    with tmpdir() as compressed, tmpdir() as decompressed, tmpdir() as dict:
+        info = TARGET_INFO[args.TARGET]
+
+        if info.input_type == InputType.DICTIONARY_DATA:
+            number = max(args.number, 1000)
+        else:
+            number = args.number
+        cmd = [
+            args.decodecorpus,
+            '-n{}'.format(args.number),
+            '-p{}/'.format(compressed),
+            '-o{}'.format(decompressed),
+        ]
+
+        if info.frame_type == FrameType.BLOCK:
+            cmd += [
+                '--gen-blocks',
+                '--max-block-size-log={}'.format(min(args.max_size_log, 17))
             ]
+        else:
+            cmd += ['--max-content-size-log={}'.format(args.max_size_log)]
 
-            info = TARGET_INFO[args.TARGET]
-            if info.frame_type == FrameType.BLOCK:
-                cmd += [
-                    '--gen-blocks',
-                    '--max-block-size-log={}'.format(min(args.max_size_log, 17))
+        print(' '.join(cmd))
+        subprocess.check_call(cmd)
+
+        if info.input_type == InputType.RAW_DATA:
+            print('using decompressed data in {}'.format(decompressed))
+            samples = decompressed
+        elif info.input_type == InputType.COMPRESSED_DATA:
+            print('using compressed data in {}'.format(compressed))
+            samples = compressed
+        else:
+            assert info.input_type == InputType.DICTIONARY_DATA
+            print('making dictionary data from {}'.format(decompressed))
+            samples = dict
+            min_dict_size_log = 9
+            max_dict_size_log = max(min_dict_size_log + 1, args.max_size_log)
+            for dict_size_log in range(min_dict_size_log, max_dict_size_log):
+                dict_size = 1 << dict_size_log
+                cmd = [
+                    args.zstd,
+                    '--train',
+                    '-r', decompressed,
+                    '--maxdict={}'.format(dict_size),
+                    '-o', abs_join(dict, '{}.zstd-dict'.format(dict_size))
                 ]
-            else:
-                cmd += ['--max-content-size-log={}'.format(args.max_size_log)]
+                print(' '.join(cmd))
+                subprocess.check_call(cmd)
 
-            print(' '.join(cmd))
-            subprocess.check_call(cmd)
-
-            if info.input_type == InputType.RAW_DATA:
-                print('using decompressed data in {}'.format(decompressed))
-                samples = decompressed
-            else:
-                assert info.input_type == InputType.COMPRESSED_DATA
-                print('using compressed data in {}'.format(compressed))
-                samples = compressed
-
-            # Copy the samples over and prepend the RNG seeds
-            for name in os.listdir(samples):
-                samplename = abs_join(samples, name)
-                outname = abs_join(seed, name)
-                with open(samplename, 'rb') as sample:
-                    with open(outname, 'wb') as out:
-                        CHUNK_SIZE = 131072
+        # Copy the samples over and prepend the RNG seeds
+        for name in os.listdir(samples):
+            samplename = abs_join(samples, name)
+            outname = abs_join(seed, name)
+            with open(samplename, 'rb') as sample:
+                with open(outname, 'wb') as out:
+                    CHUNK_SIZE = 131072
+                    chunk = sample.read(CHUNK_SIZE)
+                    while len(chunk) > 0:
+                        out.write(chunk)
                         chunk = sample.read(CHUNK_SIZE)
-                        while len(chunk) > 0:
-                            out.write(chunk)
-                            chunk = sample.read(CHUNK_SIZE)
     return 0
 
 

From 60205fec02ce1ba3cd4d7e828c843f8003fc48ae Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Fri, 1 Nov 2019 15:33:39 -0700
Subject: [PATCH 26/27] Fix 2 bugs in dictionary loading

* Silently skip dictionaries less than 8 bytes, unless using `ZSTD_dct_fullDict`.
  This changes the compressor, which silently skips dictionaries <= 8 bytes.
* Allow repcodes that are equal to the dictionary content size, since it is in bounds.
---
 lib/compress/zstd_compress.c     | 13 ++++++++-----
 lib/decompress/zstd_decompress.c |  4 ++--
 tests/fuzz/Makefile              |  2 +-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index a8856cd1b..35346b92c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2771,7 +2771,7 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym
 /*! ZSTD_loadZstdDictionary() :
  * @return : dictID, or an error code
  *  assumptions : magic number supposed already checked
- *                dictSize supposed > 8
+ *                dictSize supposed >= 8
  */
 static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
                                       ZSTD_matchState_t* ms,
@@ -2788,7 +2788,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
     size_t dictID;
 
     ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
-    assert(dictSize > 8);
+    assert(dictSize >= 8);
     assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY);
 
     dictPtr += 4;   /* skip magic number */
@@ -2890,7 +2890,10 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
                                void* workspace)
 {
     DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
-    if ((dict==NULL) || (dictSize<=8)) return 0;
+    if ((dict==NULL) || (dictSize<8)) {
+        RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong);
+        return 0;
+    }
 
     ZSTD_reset_compressedBlockState(bs);
 
@@ -2942,7 +2945,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
 
     FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize,
                                      ZSTDcrp_makeClean, zbuff) );
-    {   size_t const dictID = cdict ? 
+    {   size_t const dictID = cdict ?
                 ZSTD_compress_insertDictionary(
                         cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                         &cctx->workspace, params, cdict->dictContent, cdict->dictContentSize,
@@ -3219,7 +3222,7 @@ static size_t ZSTD_initCDict_internal(
         ZSTDirp_reset,
         ZSTD_resetTarget_CDict));
     /* (Maybe) load the dictionary
-     * Skips loading the dictionary if it is <= 8 bytes.
+     * Skips loading the dictionary if it is < 8 bytes.
      */
     {   ZSTD_CCtx_params params;
         memset(&params, 0, sizeof(params));
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index ca47a6678..dd4591b7b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1096,7 +1096,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
         size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
         for (i=0; i<3; i++) {
             U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
-            RETURN_ERROR_IF(rep==0 || rep >= dictContentSize,
+            RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
                             dictionary_corrupted);
             entropy->rep[i] = rep;
     }   }
@@ -1265,7 +1265,7 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
 {
     RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
     ZSTD_clearDict(dctx);
-    if (dict && dictSize >= 8) {
+    if (dict && dictSize != 0) {
         dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
         RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
         dctx->ddict = dctx->ddictLocal;
diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile
index 41b634cab..f66dadef0 100644
--- a/tests/fuzz/Makefile
+++ b/tests/fuzz/Makefile
@@ -74,7 +74,7 @@ FUZZ_TARGETS :=       \
 	dictionary_decompress \
 	zstd_frame_info \
 	simple_compress \
-	dict_loader
+	dictionary_loader
 
 all: $(FUZZ_TARGETS)
 

From 305ac2653ea6ab761f4940aa31e284b816d4b00b Mon Sep 17 00:00:00 2001
From: Yann Collet <cyan@fb.com>
Date: Mon, 4 Nov 2019 10:59:59 -0800
Subject: [PATCH 27/27] update table benchmark with zstd v1.4.4

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9c5f92013..c94c7882a 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ on the [Silesia compression corpus].
 
 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.4.0 -1**       | 2.884 |   530 MB/s |  1360 MB/s |
+| **zstd 1.4.4 -1**       | 2.884 |   520 MB/s |  1600 MB/s |
 | zlib 1.2.11 -1          | 2.743 |   110 MB/s |   440 MB/s |
 | brotli 1.0.7 -0         | 2.701 |   430 MB/s |   470 MB/s |
 | quicklz 1.5.0 -1        | 2.238 |   600 MB/s |   800 MB/s |