From ece465644be7c45449bc4c06170afafa53d37119 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 29 Aug 2019 12:29:39 -0700 Subject: [PATCH 001/106] Adding api for extracting sequences from seqstore --- lib/compress/zstd_compress.c | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index cd73db13b..73f5cbff3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2190,6 +2190,77 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->longLengthID = 0; } +typedef struct { + U32 matchPos; + U32 offset; + U32 litLength; + U32 matchLength; + int rep; +} Sequence; + +static size_t ZSTD_getSequencesForOneBlock(ZSTD_CCtx* zc, ZSTD_CDict* cdict, + void* dst, size_t dstSize, + const void* src, size_t srcSize, + Sequence* outSeqs, size_t outSeqsSize) +{ + const seqStore_t* seqStore; + const seqDef* seqs; + size_t seqsSize; + + int i; int repIdx; U32 position; + + size_t blockSize = ZSTD_getBlockSize(zc); + size_t maxOutput = ZSTD_compressBound(blockSize); + + ASSERT(!ZSTD_isError(ZSTD_compressBegin_usingCDict(zc, cdict))); + ASSERT(dstSize >= maxOutput); dstSize = maxOutput; + ASSERT(srcSize >= blockSize); srcSize = blockSize; + ASSERT(!ZSTD_isError(ZSTD_compressBlock(zc, dst, dstSize, src, srcSize))); + + seqStore = ZSTD_getSeqStore(zc); + seqs = seqStore->sequencesStart; + seqsSize = seqStore->sequences - seqStore->sequencesStart; + + ASSERT(outSeqsSize >= seqsSize); outSeqsSize = seqsSize; + + for (i = 0, position = 0; i < seqsSize; ++i) { + outSeqs[i].offset = seqs[i].offset; + outSeqs[i].litLength = seqs[i].litLength; + outSeqs[i].matchLength = seqs[i].matchLength + 3 /* min match */; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (outSeqs[i].offset <= 3 /* num reps */) { + outSeqs[i].rep = 1; + repIdx = i - outSeqs[i].offset; + + if (repIdx >= 0) { + outSeqs[i].offset = outSeqs[repIdx].offset; + } + + if (repIdx == -1) { + outSeqs[i].offset = 1; + } else if (repIdx == -2) { + outSeqs[i].offset = 4; + } else if (repIdx == -3) { + outSeqs[i].offset = 8; + } + } else { + outSeqs[i].offset -= 3 /* num reps */; + } + + position += outSeqs[i].litLength; + outSeqs[i].matchPos = position; + position += outSeqs[i].matchLength; + } +} + typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) From 623b90f85d94f57ebfa0c7ae61e25dd489f27ecb Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 29 Aug 2019 13:09:42 -0700 Subject: [PATCH 002/106] Fixing ci-circle test complaints --- lib/compress/zstd_compress.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 73f5cbff3..f0b6136b7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2207,21 +2207,21 @@ static size_t ZSTD_getSequencesForOneBlock(ZSTD_CCtx* zc, ZSTD_CDict* cdict, const seqDef* seqs; size_t seqsSize; - int i; int repIdx; U32 position; + size_t i; int repIdx; size_t position; size_t blockSize = ZSTD_getBlockSize(zc); size_t maxOutput = ZSTD_compressBound(blockSize); - ASSERT(!ZSTD_isError(ZSTD_compressBegin_usingCDict(zc, cdict))); - ASSERT(dstSize >= maxOutput); dstSize = maxOutput; - ASSERT(srcSize >= blockSize); srcSize = blockSize; - ASSERT(!ZSTD_isError(ZSTD_compressBlock(zc, dst, dstSize, src, srcSize))); + assert(!ZSTD_isError(ZSTD_compressBegin_usingCDict(zc, cdict))); + assert(dstSize >= maxOutput); dstSize = maxOutput; + assert(srcSize >= blockSize); srcSize = blockSize; + assert(!ZSTD_isError(ZSTD_compressBlock(zc, dst, dstSize, src, srcSize))); seqStore = ZSTD_getSeqStore(zc); seqs = seqStore->sequencesStart; seqsSize = seqStore->sequences - seqStore->sequencesStart; - ASSERT(outSeqsSize >= seqsSize); outSeqsSize = seqsSize; + assert(outSeqsSize >= seqsSize); outSeqsSize = seqsSize; for (i = 0, position = 0; i < seqsSize; ++i) { outSeqs[i].offset = seqs[i].offset; From 5f8b0f6890e03050a3840ed996f95a5e8263c1ea Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 30 Aug 2019 09:18:44 -0700 Subject: [PATCH 003/106] Changing api to get sequences across all blocks --- lib/compress/zstd_compress.c | 150 ++++++++++++++------------ lib/compress/zstd_compress_internal.h | 8 ++ lib/zstd.h | 11 ++ tests/fuzzer.c | 5 + 4 files changed, 103 insertions(+), 71 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f0b6136b7..f8588b348 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -13,6 +13,7 @@ ***************************************/ #include /* INT_MAX */ #include /* memset */ +#include #include "cpu.h" #include "mem.h" #include "hist.h" /* HIST_countFast_wksp */ @@ -2190,77 +2191,6 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->longLengthID = 0; } -typedef struct { - U32 matchPos; - U32 offset; - U32 litLength; - U32 matchLength; - int rep; -} Sequence; - -static size_t ZSTD_getSequencesForOneBlock(ZSTD_CCtx* zc, ZSTD_CDict* cdict, - void* dst, size_t dstSize, - const void* src, size_t srcSize, - Sequence* outSeqs, size_t outSeqsSize) -{ - const seqStore_t* seqStore; - const seqDef* seqs; - size_t seqsSize; - - size_t i; int repIdx; size_t position; - - size_t blockSize = ZSTD_getBlockSize(zc); - size_t maxOutput = ZSTD_compressBound(blockSize); - - assert(!ZSTD_isError(ZSTD_compressBegin_usingCDict(zc, cdict))); - assert(dstSize >= maxOutput); dstSize = maxOutput; - assert(srcSize >= blockSize); srcSize = blockSize; - assert(!ZSTD_isError(ZSTD_compressBlock(zc, dst, dstSize, src, srcSize))); - - seqStore = ZSTD_getSeqStore(zc); - seqs = seqStore->sequencesStart; - seqsSize = seqStore->sequences - seqStore->sequencesStart; - - assert(outSeqsSize >= seqsSize); outSeqsSize = seqsSize; - - for (i = 0, position = 0; i < seqsSize; ++i) { - outSeqs[i].offset = seqs[i].offset; - outSeqs[i].litLength = seqs[i].litLength; - outSeqs[i].matchLength = seqs[i].matchLength + 3 /* min match */; - - if (i == seqStore->longLengthPos) { - if (seqStore->longLengthID == 1) { - outSeqs[i].litLength += 0x10000; - } else if (seqStore->longLengthID == 2) { - outSeqs[i].matchLength += 0x10000; - } - } - - if (outSeqs[i].offset <= 3 /* num reps */) { - outSeqs[i].rep = 1; - repIdx = i - outSeqs[i].offset; - - if (repIdx >= 0) { - outSeqs[i].offset = outSeqs[repIdx].offset; - } - - if (repIdx == -1) { - outSeqs[i].offset = 1; - } else if (repIdx == -2) { - outSeqs[i].offset = 4; - } else if (repIdx == -3) { - outSeqs[i].offset = 8; - } - } else { - outSeqs[i].offset -= 3 /* num reps */; - } - - position += outSeqs[i].litLength; - outSeqs[i].matchPos = position; - position += outSeqs[i].matchLength; - } -} - typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) @@ -2394,6 +2324,81 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params } } +static void ZSTD_copyBlockSequences(const seqStore_t* seqStore, seqDef* seqs, + ZSTD_Sequence* outSeqs, size_t seqsSize) +{ + size_t i; size_t position; int repIdx; + for (i = 0, position = 0; i < seqsSize; ++i) { + outSeqs[i].offset = seqs[i].offset; + outSeqs[i].litLength = seqs[i].litLength; + outSeqs[i].matchLength = seqs[i].matchLength + 3 /* min match */; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (outSeqs[i].offset <= 3 /* num reps */) { + outSeqs[i].rep = 1; + repIdx = i - outSeqs[i].offset; + + if (repIdx >= 0) { + outSeqs[i].offset = outSeqs[repIdx].offset; + } + + if (repIdx == -1) { + outSeqs[i].offset = 1; + } else if (repIdx == -2) { + outSeqs[i].offset = 4; + } else if (repIdx == -3) { + outSeqs[i].offset = 8; + } + } else { + outSeqs[i].offset -= 3 /* num reps */; + } + + position += outSeqs[i].litLength; + outSeqs[i].matchPos = position; + position += outSeqs[i].matchLength; + } +} + +static void ZSTD_getBlockSequences(ZSTD_CCtx* cctx, const seqStore_t* seqStore) +{ + size_t seqsSize = seqStore->sequences - seqStore->sequencesStart; + + assert(cctx->seqCollector.maxSequences > + (cctx->seqCollector.seqCurrent - cctx->seqCollector.seqStart) + seqsSize); + + ZSTD_copyBlockSequences(seqStore, seqStore->sequencesStart, + cctx->seqCollector.seqCurrent, seqsSize); + cctx->seqCollector.seqCurrent += seqsSize; +} + +size_t ZSTD_getSequences(ZSTD_CCtx* zc, const void* src, + size_t srcSize, ZSTD_Sequence* outSeqs, size_t outSeqsSize, + int level) +{ + size_t dstCapacity = ZSTD_compressBound(srcSize * sizeof(void*)); + void* dst = malloc(dstCapacity); + size_t seqsSize; + + SeqCollector seqCollector; + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqCurrent = outSeqs; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compressCCtx(zc, dst, dstCapacity, src, srcSize, level); + seqsSize = zc->seqCollector.seqCurrent - zc->seqCollector.seqStart; + + free(dst); + return seqsSize; +} /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. @@ -2438,6 +2443,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); FORWARD_IF_ERROR(cSize); + if (cctx->seqCollector.collectSequences) { + ZSTD_getBlockSequences(cctx, ZSTD_getSeqStore(cctx)); + } if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 6d623cc6b..d40d53404 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -192,6 +192,13 @@ typedef struct { size_t capacity; /* The capacity starting from `seq` pointer */ } rawSeqStore_t; +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + ZSTD_Sequence* seqCurrent; + size_t maxSequences; +} SeqCollector; + struct ZSTD_CCtx_params_s { ZSTD_format_e format; ZSTD_compressionParameters cParams; @@ -238,6 +245,7 @@ struct ZSTD_CCtx_s { XXH64_state_t xxhState; ZSTD_customMem customMem; size_t staticSize; + SeqCollector seqCollector; seqStore_t seqStore; /* sequences storage ptrs */ ldmState_t ldmState; /* long distance matching state */ diff --git a/lib/zstd.h b/lib/zstd.h index f8e95f228..782940ef5 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1072,6 +1072,14 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; +typedef struct { + unsigned int matchPos; + unsigned int offset; + unsigned int litLength; + unsigned int matchLength; + int rep; +} ZSTD_Sequence; + typedef struct { unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ @@ -1210,6 +1218,9 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, const void* src, + size_t srcSize, ZSTD_Sequence* outSeqs, size_t outSeqsSize, int level); + /*************************************** * Memory management diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 2de7c0096..09fe46959 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1960,6 +1960,11 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); } + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); + memset(CNBuffer, 0, 1000000); + assert(ZSTD_getSequences(ZSTD_createCCtx(), CNBuffer, 1000000, + compressedBuffer, 1000000, 3) == 1000000 / 131071 + 1); + /* All zeroes test (test bug #137) */ #define ZEROESLENGTH 100 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); From 25878b2de955c5dbe65dddc28d5a5ac3856ef1f6 Mon Sep 17 00:00:00 2001 From: Mike Swanson Date: Mon, 19 Aug 2019 11:11:04 -0700 Subject: [PATCH 004/106] =?UTF-8?q?[programs]=20don=E2=80=99t=20do=20chmod?= =?UTF-8?q?=20when=20coming=20from=20stdin=20or=20multiple=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #1729 --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 20e2ee2a1..185b4811e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -585,7 +585,7 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, { FILE* const f = fopen( dstFileName, "wb" ); if (f == NULL) { DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); - } else { + } else if(srcFileName != NULL && strcmp (srcFileName, stdinmark)) { chmod(dstFileName, 00600); } return f; From 7f98b46876c475d8a0ef4f895b83730d55a60f9b Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 5 Sep 2019 16:03:35 -0700 Subject: [PATCH 005/106] adding support for -O flag: multiple files into one directory for compressions (decompression to come) --- programs/fileio.c | 37 +++++++++++++++++++++---------------- programs/fileio.h | 15 ++++++++------- programs/util.c | 43 +++++++++++++++++++++++++++++++++++++++++++ programs/util.h | 4 ++++ programs/zstdcli.c | 26 +++++++++++++++++++++++++- 5 files changed, 101 insertions(+), 24 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 20e2ee2a1..f25771864 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1276,9 +1276,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, int result; stat_t statbuf; int transfer_permissions = 0; - assert(ress.srcFile != NULL); - if (ress.dstFile == NULL) { closeDstFile = 1; DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); @@ -1369,11 +1367,9 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, return result; } - -int FIO_compressFilename(FIO_prefs_t* const prefs, - const char* dstFileName, const char* srcFileName, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams) +int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName, + const char* srcFileName, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams) { cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -1416,22 +1412,30 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) /* FIO_compressMultipleFilenames() : * compress nbFiles files - * into one destination (outFileName) - * or into one file each (outFileName == NULL, but suffix != NULL). + * into either one destination (outFileName), + * or into one file each (outFileName == NULL, but suffix != NULL), + * or into a destination folder (specified with -O) */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** inFileNamesTable, unsigned nbFiles, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams) +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, + const char* outDirName, char** dstFileNamesTable, + unsigned nbFiles, const char* outFileName, + const char* suffix, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams) { + printf("compressing multiple...\n"); int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); /* init */ assert(outFileName != NULL || suffix != NULL); - - if (outFileName != NULL) { /* output into a single destination (stdout typically) */ + if (outDirName != NULL) { /* output into a particular folder */ + unsigned u; + for (u = 0; u < nbFiles; ++u) { + const char* const srcFileName = inFileNamesTable[u]; + const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix); + error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + } + } else if (outFileName != NULL) { /* output into a single destination (stdout typically) */ ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; @@ -1453,6 +1457,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } } FIO_freeCResources(ress); + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } diff --git a/programs/fileio.h b/programs/fileio.h index 096d90b5c..1ed6d1538 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -87,8 +87,9 @@ void FIO_setNotificationLevel(int level); /** FIO_compressFilename() : @return : 0 == ok; 1 == pb with src file. */ int FIO_compressFilename (FIO_prefs_t* const prefs, - const char* outfilename, const char* infilename, const char* dictFileName, - int compressionLevel, ZSTD_compressionParameters comprParams); + const char* outfilename, const char* infilename, + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters comprParams); /** FIO_decompressFilename() : @return : 0 == ok; 1 == pb with src file. */ @@ -103,11 +104,11 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis ***************************************/ /** FIO_compressMultipleFilenames() : @return : nb of missing files */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** srcNamesTable, unsigned nbFiles, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams); +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, + const char* outDirName, char** dstFileNamesTable, + unsigned nbFiles, const char* outFileName, + const char* suffix, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams); /** FIO_decompressMultipleFilenames() : @return : nb of missing or skipped files */ diff --git a/programs/util.c b/programs/util.c index 347e76986..e8b599aaa 100644 --- a/programs/util.c +++ b/programs/util.c @@ -87,6 +87,49 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } +int UTIL_createDir(const char* outDirName) { + if (UTIL_isDirectory(outDirName)) { + return 0; /* no need to create if directory already exists */ + } + int r; +#if defined(_MSC_VER) + r = _mkdir(outDirName); + if (r || !UTIL_isDirectory(outDirName)) return 1; +#else + r = mkdir(outDirName, S_IRWXU | S_IRWXG | S_IRWXO); /* dir has all permissions */ + if (r || !UTIL_isDirectory(outDirName)) return 1; +#endif + return 0; +} + +void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles, + const char* outDirName, char** dstFilenameTable) +{ + unsigned u; + char c; + c = '/'; + + /* duplicate source file table */ + for (u = 0; u < nbFiles; ++u) { + char* filename; + char* finalPath; + size_t finalPathLen; + finalPathLen = strlen(outDirName); + filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ + finalPathLen += strlen(filename); + dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); + strcpy(dstFilenameTable[u], outDirName); + strcat(dstFilenameTable[u], filename); + } +} + +void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { + unsigned u; + for (u = 0; u < nbFiles; ++u) + free(dstDirTable[u]); + free((void*)dstDirTable); +} + int UTIL_isSameFile(const char* file1, const char* file2) { #if defined(_MSC_VER) diff --git a/programs/util.h b/programs/util.h index d6e5bb550..1c1273262 100644 --- a/programs/util.h +++ b/programs/util.h @@ -127,8 +127,12 @@ int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); int UTIL_setFileStat(const char* filename, stat_t* statbuf); U32 UTIL_isDirectory(const char* infilename); +int UTIL_createDir(const char* outDirName); int UTIL_getFileStat(const char* infilename, stat_t* statbuf); int UTIL_isSameFile(const char* file1, const char* file2); +void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx, + const char* outDirName, char** dstFilenameTable); +void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 98df728a9..2d3254ae5 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -118,6 +118,7 @@ static int usage(const char* programName) #endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); + DISPLAY( " -O directory: result(s) stored into `directory`, creates one if non-existent \n"); DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); @@ -562,6 +563,7 @@ int main(int argCount, const char* argv[]) adaptMax = MAXCLEVEL, rsyncable = 0, nextArgumentIsOutFileName = 0, + nextArgumentIsOutDirName = 0, nextArgumentIsMaxDict = 0, nextArgumentIsDictID = 0, nextArgumentsAreFiles = 0, @@ -583,9 +585,11 @@ int main(int argCount, const char* argv[]) unsigned recursive = 0; unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ + char** dstFilenameTable = (char**)malloc(argCount * sizeof(char*)); unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; + const char* outDirName = NULL; const char* dictFileName = NULL; const char* suffix = ZSTD_EXTENSION; unsigned maxDictSize = g_defaultMaxDictSize; @@ -853,6 +857,9 @@ int main(int argCount, const char* argv[]) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; + /* destination directory name */ + case 'O': nextArgumentIsOutDirName=1; lastCommand=1; argument++; break; + /* limit decompression memory */ case 'M': argument++; @@ -965,6 +972,13 @@ int main(int argCount, const char* argv[]) continue; } + if (nextArgumentIsOutDirName) { + nextArgumentIsOutDirName = 0; + lastCommand = 0; + outDirName = argument; + continue; + } + /* add filename to list */ filenameTable[filenameIdx++] = argument; } @@ -1163,10 +1177,20 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; + if (outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) DISPLAY("Directory creation unsuccessful \n"); + + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } + } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); From 6beb3c0159a60a6c49334f24e32ae92df5dbba4d Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 5 Sep 2019 17:56:24 -0700 Subject: [PATCH 006/106] added decompression support --- programs/fileio.c | 30 ++++++++++++++++++++++++++---- programs/fileio.h | 5 +++++ programs/util.c | 10 +++++++--- programs/zstdcli.c | 17 +++++++---------- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index f25771864..c8a971c27 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1422,7 +1422,6 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN const char* suffix, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { - printf("compressing multiple...\n"); int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); @@ -1457,7 +1456,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + /*UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);*/ return error; } @@ -2240,14 +2239,24 @@ FIO_determineDstName(const char* srcFileName) int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, - const char* srcNamesTable[], unsigned nbFiles, + const char** srcNamesTable, unsigned nbFiles, + const char* outDirName, char** dstFileNamesTable, const char* outFileName, const char* dictFileName) { int error = 0; dRess_t ress = FIO_createDResources(prefs, dictFileName); - if (outFileName) { + if (outDirName != NULL) { /* output into a particular folder */ + unsigned u; + for (u = 0; u < nbFiles; ++u) { + const char* const srcFileName = srcNamesTable[u]; + const char* const dstFileName = FIO_determineDstName(dstFileNamesTable[u]); + if (dstFileName == NULL) { error=1; continue; } + + error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); + } + } else if (outFileName) { unsigned u; ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); @@ -2268,10 +2277,23 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); + /* UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); */ return error; } +void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) + DISPLAY("Directory creation unsuccessful \n"); + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } +} /* ************************************************************************** * .zst file info (--list command) diff --git a/programs/fileio.h b/programs/fileio.h index 1ed6d1538..1b435c5f3 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -114,9 +114,14 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN @return : nb of missing or skipped files */ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char** srcNamesTable, unsigned nbFiles, + const char* outDirName, char** dstFilenameTable, const char* outFileName, const char* dictFileName); +void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName); + /*-************************************* * Advanced stuff (should actually be hosted elsewhere) diff --git a/programs/util.c b/programs/util.c index e8b599aaa..448a78899 100644 --- a/programs/util.c +++ b/programs/util.c @@ -108,6 +108,7 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles unsigned u; char c; c = '/'; + printf("NBFILE: %u\n", nbFiles); /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { @@ -120,14 +121,17 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); strcpy(dstFilenameTable[u], outDirName); strcat(dstFilenameTable[u], filename); + printf("%s %s\n", filenameTable[u], dstFilenameTable[u]); } } void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { unsigned u; - for (u = 0; u < nbFiles; ++u) - free(dstDirTable[u]); - free((void*)dstDirTable); + for (u = 0; u < nbFiles; ++u) { + if (dstDirTable[u] != NULL) + free(dstDirTable[u]); + } + if (dstDirTable != NULL) free((void*)dstDirTable); } int UTIL_isSameFile(const char* file1, const char* file2) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 2d3254ae5..7c00b3d71 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1177,16 +1177,9 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; - if (outDirName) { - int dirResult; - dirResult = UTIL_createDir(outDirName); - if (dirResult) DISPLAY("Directory creation unsuccessful \n"); + if (outDirName) + FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); - UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); - if (outFileName) { - outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ - } - } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else @@ -1205,10 +1198,14 @@ int main(int argCount, const char* argv[]) } } FIO_setMemLimit(prefs, memLimit); + + if (outDirName) + FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else - operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName); #else DISPLAY("Decompression not supported \n"); #endif From a9c807a9489cc1c313ab027c8206fb95eb1adc0e Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 6 Sep 2019 10:17:04 -0700 Subject: [PATCH 007/106] kill memory leaks, cleanup, fix some dumb bugs --- programs/fileio.c | 7 ++----- programs/util.c | 14 ++++++-------- programs/zstdcli.c | 11 +++++++---- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c8a971c27..194ea85c4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1390,10 +1390,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) { static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - size_t const sfnSize = strlen(srcFileName); size_t const suffixSize = strlen(suffix); - if (dfnbCapacity <= sfnSize+suffixSize+1) { /* resize buffer for dstName */ free(dstFileNameBuffer); @@ -1405,7 +1403,6 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) assert(dstFileNameBuffer != NULL); memcpy(dstFileNameBuffer, srcFileName, sfnSize); memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */); - return dstFileNameBuffer; } @@ -1456,7 +1453,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - /*UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);*/ + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } @@ -2277,7 +2274,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); - /* UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); */ + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } diff --git a/programs/util.c b/programs/util.c index 448a78899..3b134f9cd 100644 --- a/programs/util.c +++ b/programs/util.c @@ -87,11 +87,12 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } -int UTIL_createDir(const char* outDirName) { - if (UTIL_isDirectory(outDirName)) { - return 0; /* no need to create if directory already exists */ - } +int UTIL_createDir(const char* outDirName) +{ int r; + if (UTIL_isDirectory(outDirName)) + return 0; /* no need to create if directory already exists */ + #if defined(_MSC_VER) r = _mkdir(outDirName); if (r || !UTIL_isDirectory(outDirName)) return 1; @@ -108,20 +109,17 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles unsigned u; char c; c = '/'; - printf("NBFILE: %u\n", nbFiles); /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { char* filename; - char* finalPath; size_t finalPathLen; finalPathLen = strlen(outDirName); filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ finalPathLen += strlen(filename); - dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); + dstFilenameTable[u] = (char*) malloc((finalPathLen+5) * sizeof(char)); /* extra 1 bit for \0, extra 4 for .zst if compressing*/ strcpy(dstFilenameTable[u], outDirName); strcat(dstFilenameTable[u], filename); - printf("%s %s\n", filenameTable[u], dstFilenameTable[u]); } } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7c00b3d71..9029f6344 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -585,7 +585,7 @@ int main(int argCount, const char* argv[]) unsigned recursive = 0; unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ - char** dstFilenameTable = (char**)malloc(argCount * sizeof(char*)); + char** dstFilenameTable; unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; @@ -1177,8 +1177,10 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; - if (outDirName) + if (outDirName) { + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); @@ -1199,9 +1201,10 @@ int main(int argCount, const char* argv[]) } FIO_setMemLimit(prefs, memLimit); - if (outDirName) + if (outDirName) { + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); - + } if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else From 30bff50e06d84d1d426fee62acd34223abc69f1b Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 6 Sep 2019 11:10:53 -0700 Subject: [PATCH 008/106] fixes for tests and segfault --- programs/fileio.c | 22 ++++++---------------- programs/fileio.h | 4 ---- programs/util.c | 14 ++++++++++++++ programs/util.h | 3 +++ programs/zstdcli.c | 5 +++-- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 194ea85c4..cc8809f94 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1453,7 +1453,9 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + if (outDirName) + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + return error; } @@ -2274,24 +2276,12 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + if (outDirName) + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); + return error; } -void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName) { - int dirResult; - dirResult = UTIL_createDir(outDirName); - if (dirResult) - DISPLAY("Directory creation unsuccessful \n"); - - UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); - if (outFileName) { - outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ - } -} - /* ************************************************************************** * .zst file info (--list command) ***************************************************************************/ diff --git a/programs/fileio.h b/programs/fileio.h index 1b435c5f3..40cd8985f 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -118,10 +118,6 @@ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char* outFileName, const char* dictFileName); -void FIO_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName); - /*-************************************* * Advanced stuff (should actually be hosted elsewhere) diff --git a/programs/util.c b/programs/util.c index 3b134f9cd..f00bfd608 100644 --- a/programs/util.c +++ b/programs/util.c @@ -123,6 +123,20 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles } } +void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) + UTIL_DISPLAYLEVEL(1, "Directory creation unsuccessful\n"); + + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } +} + void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { unsigned u; for (u = 0; u < nbFiles; ++u) { diff --git a/programs/util.h b/programs/util.h index 1c1273262..9615504ca 100644 --- a/programs/util.h +++ b/programs/util.h @@ -133,6 +133,9 @@ int UTIL_isSameFile(const char* file1, const char* file2); void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx, const char* outDirName, char** dstFilenameTable); void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); +void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, + const char** filenameTable, unsigned filenameIdx, + const char* outFileName, const char* outDirName); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9029f6344..606c162f7 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1178,8 +1178,9 @@ int main(int argCount, const char* argv[]) if (adaptMax < cLevel) cLevel = adaptMax; if (outDirName) { + printf("ok\n"); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); } if ((filenameIdx==1) && outFileName) @@ -1203,7 +1204,7 @@ int main(int argCount, const char* argv[]) if (outDirName) { dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - FIO_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); } if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); From 62616c4d90557ab070ddd7677840e882cf0e34b9 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Fri, 6 Sep 2019 13:20:50 -0700 Subject: [PATCH 009/106] fixes for windows compilation --- programs/util.c | 7 +++++-- programs/zstdcli.c | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/programs/util.c b/programs/util.c index f00bfd608..920601fba 100644 --- a/programs/util.c +++ b/programs/util.c @@ -20,6 +20,9 @@ extern "C" { #include #include +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) +#include /* needed for _mkdir in windows */ +#endif int UTIL_fileExist(const char* filename) { @@ -93,7 +96,7 @@ int UTIL_createDir(const char* outDirName) if (UTIL_isDirectory(outDirName)) return 0; /* no need to create if directory already exists */ -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) r = _mkdir(outDirName); if (r || !UTIL_isDirectory(outDirName)) return 1; #else @@ -112,7 +115,7 @@ void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { - char* filename; + const char* filename; size_t finalPathLen; finalPathLen = strlen(outDirName); filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 606c162f7..5fc6b8aa8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1181,6 +1181,8 @@ int main(int argCount, const char* argv[]) printf("ok\n"); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + } else { + dstFilenameTable = NULL; } if ((filenameIdx==1) && outFileName) @@ -1205,7 +1207,10 @@ int main(int argCount, const char* argv[]) if (outDirName) { dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + } else { + dstFilenameTable = NULL; } + if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else From 9e7bb55e14b112e514a0e8d5ac7f19fd64f353c1 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 9 Sep 2019 20:04:46 -0700 Subject: [PATCH 010/106] Addressing comments --- lib/compress/zstd_compress.c | 156 +++++++++++++------------- lib/compress/zstd_compress_internal.h | 2 +- lib/zstd.h | 4 +- tests/fuzzer.c | 4 +- 4 files changed, 80 insertions(+), 86 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f8588b348..56da1664e 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -13,7 +13,6 @@ ***************************************/ #include /* INT_MAX */ #include /* memset */ -#include #include "cpu.h" #include "mem.h" #include "hist.h" /* HIST_countFast_wksp */ @@ -2265,6 +2264,77 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) return ZSTDbss_compress; } +static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +{ + const seqStore_t* seqStore = ZSTD_getSeqStore(zc); + const seqDef* seqs = seqStore->sequencesStart; + size_t seqsSize = seqStore->sequences - seqs; + + ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + size_t i; size_t position; int repIdx; + + assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); + for (i = 0, position = 0; i < seqsSize; ++i) { + outSeqs[i].offset = seqs[i].offset; + outSeqs[i].litLength = seqs[i].litLength; + outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (outSeqs[i].offset <= ZSTD_REP_NUM) { + outSeqs[i].rep = 1; + repIdx = i - outSeqs[i].offset; + + if (repIdx >= 0) { + outSeqs[i].offset = outSeqs[repIdx].offset; + } + + if (repIdx == -1) { + outSeqs[i].offset = 1; + } else if (repIdx == -2) { + outSeqs[i].offset = 4; + } else if (repIdx == -3) { + outSeqs[i].offset = 8; + } + } else { + outSeqs[i].offset -= ZSTD_REP_NUM; + } + + position += outSeqs[i].litLength; + outSeqs[i].matchPos = position; + position += outSeqs[i].matchLength; + } + zc->seqCollector.seqIndex += seqsSize; +} + +/* We call compress2() and collect sequences after each block + * compression. The function stores the ZSTD_Sequences in outSeqs + * and returns the number of collected sequences from all blocks. + */ +size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) +{ + const size_t dstCapacity = ZSTD_compressBound(srcSize * sizeof(void*)); + void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); + + SeqCollector seqCollector; + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_free(dst, ZSTD_defaultCMem); + return zc->seqCollector.seqIndex; +} + static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) @@ -2288,6 +2358,10 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->bmi2); + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + } + out: if (!ZSTD_isError(cSize) && cSize != 0) { /* confirm repcodes and entropy tables when emitting a compressed block */ @@ -2324,82 +2398,6 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params } } -static void ZSTD_copyBlockSequences(const seqStore_t* seqStore, seqDef* seqs, - ZSTD_Sequence* outSeqs, size_t seqsSize) -{ - size_t i; size_t position; int repIdx; - for (i = 0, position = 0; i < seqsSize; ++i) { - outSeqs[i].offset = seqs[i].offset; - outSeqs[i].litLength = seqs[i].litLength; - outSeqs[i].matchLength = seqs[i].matchLength + 3 /* min match */; - - if (i == seqStore->longLengthPos) { - if (seqStore->longLengthID == 1) { - outSeqs[i].litLength += 0x10000; - } else if (seqStore->longLengthID == 2) { - outSeqs[i].matchLength += 0x10000; - } - } - - if (outSeqs[i].offset <= 3 /* num reps */) { - outSeqs[i].rep = 1; - repIdx = i - outSeqs[i].offset; - - if (repIdx >= 0) { - outSeqs[i].offset = outSeqs[repIdx].offset; - } - - if (repIdx == -1) { - outSeqs[i].offset = 1; - } else if (repIdx == -2) { - outSeqs[i].offset = 4; - } else if (repIdx == -3) { - outSeqs[i].offset = 8; - } - } else { - outSeqs[i].offset -= 3 /* num reps */; - } - - position += outSeqs[i].litLength; - outSeqs[i].matchPos = position; - position += outSeqs[i].matchLength; - } -} - -static void ZSTD_getBlockSequences(ZSTD_CCtx* cctx, const seqStore_t* seqStore) -{ - size_t seqsSize = seqStore->sequences - seqStore->sequencesStart; - - assert(cctx->seqCollector.maxSequences > - (cctx->seqCollector.seqCurrent - cctx->seqCollector.seqStart) + seqsSize); - - ZSTD_copyBlockSequences(seqStore, seqStore->sequencesStart, - cctx->seqCollector.seqCurrent, seqsSize); - cctx->seqCollector.seqCurrent += seqsSize; -} - -size_t ZSTD_getSequences(ZSTD_CCtx* zc, const void* src, - size_t srcSize, ZSTD_Sequence* outSeqs, size_t outSeqsSize, - int level) -{ - size_t dstCapacity = ZSTD_compressBound(srcSize * sizeof(void*)); - void* dst = malloc(dstCapacity); - size_t seqsSize; - - SeqCollector seqCollector; - seqCollector.collectSequences = 1; - seqCollector.seqStart = outSeqs; - seqCollector.seqCurrent = outSeqs; - seqCollector.maxSequences = outSeqsSize; - zc->seqCollector = seqCollector; - - ZSTD_compressCCtx(zc, dst, dstCapacity, src, srcSize, level); - seqsSize = zc->seqCollector.seqCurrent - zc->seqCollector.seqStart; - - free(dst); - return seqsSize; -} - /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -2443,10 +2441,6 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); FORWARD_IF_ERROR(cSize); - if (cctx->seqCollector.collectSequences) { - ZSTD_getBlockSequences(cctx, ZSTD_getSeqStore(cctx)); - } - if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); FORWARD_IF_ERROR(cSize); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index d40d53404..e3ed93eb4 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -195,7 +195,7 @@ typedef struct { typedef struct { int collectSequences; ZSTD_Sequence* seqStart; - ZSTD_Sequence* seqCurrent; + size_t seqIndex; size_t maxSequences; } SeqCollector; diff --git a/lib/zstd.h b/lib/zstd.h index 782940ef5..b2c66e755 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1218,8 +1218,8 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, const void* src, - size_t srcSize, ZSTD_Sequence* outSeqs, size_t outSeqsSize, int level); +ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize); /*************************************** diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 09fe46959..fdf6960bc 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1962,8 +1962,8 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); memset(CNBuffer, 0, 1000000); - assert(ZSTD_getSequences(ZSTD_createCCtx(), CNBuffer, 1000000, - compressedBuffer, 1000000, 3) == 1000000 / 131071 + 1); + assert(ZSTD_getSequences(ZSTD_createCCtx(), compressedBuffer, 1000000, + CNBuffer, 1000000) == 1000000 / 131071 + 1); /* All zeroes test (test bug #137) */ #define ZEROESLENGTH 100 From e3c582591855a2e70a8c88976a9f05c6b1a0ea8e Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Tue, 10 Sep 2019 10:06:02 -0700 Subject: [PATCH 011/106] Fizing litLength == 0 case --- lib/compress/zstd_compress.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 56da1664e..83bcb52dd 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2291,11 +2291,22 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) outSeqs[i].rep = 1; repIdx = i - outSeqs[i].offset; + /* Not first block */ if (repIdx >= 0) { - outSeqs[i].offset = outSeqs[repIdx].offset; - } - - if (repIdx == -1) { + /* Special case where litLength == 0 */ + if (outSeqs[i].litLength == 0) { + /* When the offset is 3 */ + if (outSeqs[i].offset > 2) { + outSeqs[i].offset = outSeqs[repIdx - 1].offset - 1; + /* When the offset is either 1 or 2 */ + } else { + outSeqs[i].offset = outSeqs[repIdx - 1].offset; + } + } else { + outSeqs[i].offset = outSeqs[repIdx].offset; + } + /* First block */ + } else if (repIdx == -1) { outSeqs[i].offset = 1; } else if (repIdx == -2) { outSeqs[i].offset = 4; From 47199480da07531a1e2b982483e08e23eff54970 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Tue, 10 Sep 2019 13:18:59 -0700 Subject: [PATCH 012/106] Cleaning up parsing per suggestion --- lib/compress/zstd_compress.c | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 83bcb52dd..b4fdf3872 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2291,27 +2291,17 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) outSeqs[i].rep = 1; repIdx = i - outSeqs[i].offset; - /* Not first block */ - if (repIdx >= 0) { - /* Special case where litLength == 0 */ - if (outSeqs[i].litLength == 0) { - /* When the offset is 3 */ - if (outSeqs[i].offset > 2) { - outSeqs[i].offset = outSeqs[repIdx - 1].offset - 1; - /* When the offset is either 1 or 2 */ - } else { - outSeqs[i].offset = outSeqs[repIdx - 1].offset; - } + if (outSeqs[i].litLength == 0) { + if (outSeqs[i].offset < 3) { + --repIdx; } else { - outSeqs[i].offset = outSeqs[repIdx].offset; + repIdx = i - 1; } - /* First block */ - } else if (repIdx == -1) { - outSeqs[i].offset = 1; - } else if (repIdx == -2) { - outSeqs[i].offset = 4; - } else if (repIdx == -3) { - outSeqs[i].offset = 8; + } + assert(repIdx >= -3); + outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; + if (outSeqs[i].offset == 3) { + --outSeqs[i].offset; } } else { outSeqs[i].offset -= ZSTD_REP_NUM; From 1407919d132f1b29998939705fc3a7f0240f3a7c Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Tue, 10 Sep 2019 15:10:50 -0700 Subject: [PATCH 013/106] Addressing comments on parsing --- lib/compress/zstd_compress.c | 5 +++-- lib/zstd.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b4fdf3872..849a9f42c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2288,7 +2288,7 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) } if (outSeqs[i].offset <= ZSTD_REP_NUM) { - outSeqs[i].rep = 1; + outSeqs[i].rep = outSeqs[i].offset; repIdx = i - outSeqs[i].offset; if (outSeqs[i].litLength == 0) { @@ -2297,10 +2297,11 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) } else { repIdx = i - 1; } + ++outSeqs[i].rep; } assert(repIdx >= -3); outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; - if (outSeqs[i].offset == 3) { + if (outSeqs[i].offset == 4) { --outSeqs[i].offset; } } else { diff --git a/lib/zstd.h b/lib/zstd.h index b2c66e755..97feb77d5 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1077,7 +1077,7 @@ typedef struct { unsigned int offset; unsigned int litLength; unsigned int matchLength; - int rep; + unsigned int rep; } ZSTD_Sequence; typedef struct { From 4607f80f5adba93b429cc801d62604a55678bf54 Mon Sep 17 00:00:00 2001 From: Dmitri Shubin Date: Mon, 16 Sep 2019 11:36:37 +0300 Subject: [PATCH 014/106] Don't hardcode installation directories in CMakeLists.txt Use paths provided by GNUInstallDirs module instead. --- build/cmake/lib/CMakeLists.txt | 8 ++++---- build/cmake/programs/CMakeLists.txt | 26 ++++++++++++++------------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 5c0802bcc..77b389cae 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -133,8 +133,8 @@ endif () if (UNIX) # pkg-config set(PREFIX "${CMAKE_INSTALL_PREFIX}") - set(LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") - set(INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include") + set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}") + set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}") set(VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}") add_custom_target(libzstd.pc ALL ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc" @@ -152,10 +152,10 @@ install(FILES ${LIBRARY_DIR}/dictBuilder/zdict.h ${LIBRARY_DIR}/dictBuilder/cover.h ${LIBRARY_DIR}/common/zstd_errors.h - DESTINATION "include") + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") if (ZSTD_BUILD_SHARED) - install(TARGETS libzstd_shared RUNTIME DESTINATION "bin" + install(TARGETS libzstd_shared RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") endif() diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt index 50408bd9b..8afd83503 100644 --- a/build/cmake/programs/CMakeLists.txt +++ b/build/cmake/programs/CMakeLists.txt @@ -31,15 +31,15 @@ target_link_libraries(zstd libzstd_static) if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)") target_link_libraries(zstd rt) endif () -install(TARGETS zstd RUNTIME DESTINATION "bin") +install(TARGETS zstd RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") if (UNIX) add_custom_target(zstdcat ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdcat DEPENDS zstd COMMENT "Creating zstdcat symlink") add_custom_target(unzstd ALL ${CMAKE_COMMAND} -E create_symlink zstd unzstd DEPENDS zstd COMMENT "Creating unzstd symlink") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "bin") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "bin") - install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "bin") - install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "bin") + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat DESTINATION "${CMAKE_INSTALL_BINDIR}") + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd DESTINATION "${CMAKE_INSTALL_BINDIR}") + install(PROGRAMS ${PROGRAMS_DIR}/zstdgrep DESTINATION "${CMAKE_INSTALL_BINDIR}") + install(PROGRAMS ${PROGRAMS_DIR}/zstdless DESTINATION "${CMAKE_INSTALL_BINDIR}") add_custom_target(zstd.1 ALL ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 . @@ -56,14 +56,16 @@ if (UNIX) # Define MAN_INSTALL_DIR if necessary if (MAN_INSTALL_DIR) else () - set(MAN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/share/man/man1) + set(MAN_INSTALL_DIR ${CMAKE_INSTALL_MANDIR}/man1) endif () - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "${MAN_INSTALL_DIR}") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "${MAN_INSTALL_DIR}") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "${MAN_INSTALL_DIR}") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}") + install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 + ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 + ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 + ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 + ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 + DESTINATION "${MAN_INSTALL_DIR}") add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c) target_link_libraries(zstd-frugal libzstd_static) @@ -79,7 +81,7 @@ if (ZSTD_MULTITHREAD_SUPPORT) target_link_libraries(zstd ${THREADS_LIBS}) add_custom_target(zstdmt ALL ${CMAKE_COMMAND} -E create_symlink zstd zstdmt DEPENDS zstd COMMENT "Creating zstdmt symlink") - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "bin") + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdmt DESTINATION "${CMAKE_INSTALL_BINDIR}") endif () endif () From bff6072e3a691a47638153d9daa1cd9de7c119b4 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 16 Sep 2019 08:26:21 -0700 Subject: [PATCH 015/106] Bailing early when collecting sequences and documentation --- lib/compress/zstd_compress.c | 13 +++++-------- lib/zstd.h | 16 +++++++++++----- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 849a9f42c..833ae8383 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2315,10 +2315,6 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) zc->seqCollector.seqIndex += seqsSize; } -/* We call compress2() and collect sequences after each block - * compression. The function stores the ZSTD_Sequences in outSeqs - * and returns the number of collected sequences from all blocks. - */ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize) { @@ -2351,6 +2347,11 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } } + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + return 0; + } + /* encode sequences and literals */ cSize = ZSTD_compressSequences(&zc->seqStore, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, @@ -2360,10 +2361,6 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->bmi2); - if (zc->seqCollector.collectSequences) { - ZSTD_copyBlockSequences(zc); - } - out: if (!ZSTD_isError(cSize) && cSize != 0) { /* confirm repcodes and entropy tables when emitting a compressed block */ diff --git a/lib/zstd.h b/lib/zstd.h index 97feb77d5..3e737d8fe 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1073,11 +1073,11 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; typedef struct { - unsigned int matchPos; - unsigned int offset; - unsigned int litLength; - unsigned int matchLength; - unsigned int rep; + unsigned int matchPos; /* match pos in dst */ + unsigned int offset; /* offset taking into account rep (different from seqdef) */ + unsigned int litLength; /* literal length */ + unsigned int matchLength; /* match length */ + unsigned int rep; /* 0 when seq not rep and seqDef.offset otherwise */ } ZSTD_Sequence; typedef struct { @@ -1218,6 +1218,12 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +/*! ZSTD_getSequences() : + * Extract sequences from the sequence store + * zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2 + * @return : number of sequences extracted + */ ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize); From 1f93be0f6dbb9bad2103b71a249aedee8b5dc2ad Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 16 Sep 2019 13:35:45 -0700 Subject: [PATCH 016/106] Handling memory leak and potential side effect --- tests/fuzzer.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index fdf6960bc..b11a3cc06 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1960,10 +1960,15 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); } - DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); - memset(CNBuffer, 0, 1000000); - assert(ZSTD_getSequences(ZSTD_createCCtx(), compressedBuffer, 1000000, - CNBuffer, 1000000) == 1000000 / 131071 + 1); + { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + assert(cctx != NULL); + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); + memset(CNBuffer, 0, 1000000); + assert(ZSTD_getSequences(cctx, compressedBuffer, 1000000, + CNBuffer, 1000000) == 1000000 / 131071 + 1); + ZSTD_freeCCtx(cctx); + } /* All zeroes test (test bug #137) */ #define ZEROESLENGTH 100 From 76fea3fb9922c68a45159aa68b536ac196f31ab7 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 16 Sep 2019 14:02:23 -0700 Subject: [PATCH 017/106] Resolving appveyor test failure implicit conversion --- lib/compress/zstd_compress.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3fe84b6bc..acb3b15e8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2290,13 +2290,13 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) if (outSeqs[i].offset <= ZSTD_REP_NUM) { outSeqs[i].rep = outSeqs[i].offset; - repIdx = i - outSeqs[i].offset; + repIdx = (unsigned int)i - outSeqs[i].offset; if (outSeqs[i].litLength == 0) { if (outSeqs[i].offset < 3) { --repIdx; } else { - repIdx = i - 1; + repIdx = (unsigned int)i - 1; } ++outSeqs[i].rep; } @@ -2310,7 +2310,7 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) } position += outSeqs[i].litLength; - outSeqs[i].matchPos = position; + outSeqs[i].matchPos = (unsigned int)position; position += outSeqs[i].matchLength; } zc->seqCollector.seqIndex += seqsSize; From 3cacc0a30bb3f39af6f4cb7f2ebca9ace9f45aff Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Tue, 17 Sep 2019 17:44:08 -0700 Subject: [PATCH 018/106] Casting void pointer to ZSTD_Sequence pointer --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 62263fd1d..a513e1b40 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1965,7 +1965,7 @@ static int basicUnitTests(U32 const seed, double compressibility) assert(cctx != NULL); DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); memset(CNBuffer, 0, 1000000); - assert(ZSTD_getSequences(cctx, compressedBuffer, 1000000, + assert(ZSTD_getSequences(cctx, (ZSTD_Sequence*)compressedBuffer, 1000000, CNBuffer, 1000000) == 1000000 / 131071 + 1); ZSTD_freeCCtx(cctx); } From ae6d0e64ae0fcd66a6c1d2061a017bc7521ac7c0 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 19 Sep 2019 15:25:20 -0700 Subject: [PATCH 019/106] Addressing comments --- lib/compress/zstd_compress.c | 9 +++------ lib/zstd.h | 20 +++++++++++++++----- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index acb3b15e8..ce352acde 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2302,7 +2302,7 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) } assert(repIdx >= -3); outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; - if (outSeqs[i].offset == 4) { + if (outSeqs[i].rep == 4) { --outSeqs[i].offset; } } else { @@ -2319,9 +2319,6 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize) { - const size_t dstCapacity = ZSTD_compressBound(srcSize * sizeof(void*)); - void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); - SeqCollector seqCollector; seqCollector.collectSequences = 1; seqCollector.seqStart = outSeqs; @@ -2329,8 +2326,8 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, seqCollector.maxSequences = outSeqsSize; zc->seqCollector = seqCollector; - ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); - ZSTD_free(dst, ZSTD_defaultCMem); + /* We never write to dst when collecing sequences so setting dst = src is harmless */ + ZSTD_compress2(zc, (void*)src, srcSize, src, srcSize); return zc->seqCollector.seqIndex; } diff --git a/lib/zstd.h b/lib/zstd.h index 217a6d350..836aa7238 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1078,11 +1078,21 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; typedef struct { - unsigned int matchPos; /* match pos in dst */ - unsigned int offset; /* offset taking into account rep (different from seqdef) */ - unsigned int litLength; /* literal length */ - unsigned int matchLength; /* match length */ - unsigned int rep; /* 0 when seq not rep and seqDef.offset otherwise */ + unsigned int matchPos; /* Match pos in dst */ + /* If seqDef.offset > 3, then this is seqDef.offset - 3 + * If seqDef.offset < 3, then this is the corresponding repeat offset + * But if seqDef.offset < 3 and litLength == 0, this is the + * repeat offset before the corresponding repeat offset + * And if seqDef.offset == 3 and litLength == 0, this is the + * most recent repeat offset - 1 + */ + unsigned int offset; + unsigned int litLength; /* Literal length */ + unsigned int matchLength; /* Match length */ + /* 0 when seq not rep and seqDef.offset otherwise + * when litLength == 0 this will be <= 4, otherwise <= 3 like normal + */ + unsigned int rep; } ZSTD_Sequence; typedef struct { From f3c4fd17e30465a4ec90152c9f858dc8ff674b7f Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 20 Sep 2019 15:50:58 -0700 Subject: [PATCH 020/106] Passing in dummy dst buffer of compressbound(srcSize) --- lib/compress/zstd_compress.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ce352acde..e7ec1d61f 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2319,6 +2319,9 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize) { + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); + SeqCollector seqCollector; seqCollector.collectSequences = 1; seqCollector.seqStart = outSeqs; @@ -2326,8 +2329,8 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, seqCollector.maxSequences = outSeqsSize; zc->seqCollector = seqCollector; - /* We never write to dst when collecing sequences so setting dst = src is harmless */ - ZSTD_compress2(zc, (void*)src, srcSize, src, srcSize); + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_free(dst, ZSTD_defaultCMem); return zc->seqCollector.seqIndex; } From be0bebd24e08833f757daaee3c172514c0cc811f Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 23 Sep 2019 15:08:18 -0700 Subject: [PATCH 021/106] Adding test and null check for malloc --- lib/compress/zstd_compress.c | 4 +++- tests/fuzzer.c | 26 ++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e7ec1d61f..8eb292894 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2321,8 +2321,10 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, { const size_t dstCapacity = ZSTD_compressBound(srcSize); void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); - SeqCollector seqCollector; + + assert(dst != NULL); + seqCollector.collectSequences = 1; seqCollector.seqStart = outSeqs; seqCollector.seqIndex = 0; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index a513e1b40..349f0d19b 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1962,11 +1962,29 @@ static int basicUnitTests(U32 const seed, double compressibility) { ZSTD_CCtx* cctx = ZSTD_createCCtx(); - assert(cctx != NULL); + size_t zerosLength = ZSTD_BLOCKSIZE_MAX * 2 - 1; + size_t expectedOffsets[] = {1, 1}; + size_t expectedLitLengths[] = {2, 1}; + size_t expectedMatchLengths[] = {ZSTD_BLOCKSIZE_MAX - 2, ZSTD_BLOCKSIZE_MAX - 2}; + size_t expectedReps[] = {1, 1}; + size_t expectedMatchPos[] = {2, 1}; + size_t expectedSequencesSize = 2; + size_t sequencesSize; + size_t i = 0; + ZSTD_Sequence* sequences = (ZSTD_Sequence*)compressedBuffer; DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); - memset(CNBuffer, 0, 1000000); - assert(ZSTD_getSequences(cctx, (ZSTD_Sequence*)compressedBuffer, 1000000, - CNBuffer, 1000000) == 1000000 / 131071 + 1); + assert(cctx != NULL); + memset(CNBuffer, 0, zerosLength); + sequencesSize = ZSTD_getSequences(cctx, sequences, 10, + CNBuffer, zerosLength); + assert(sequencesSize == expectedSequencesSize); + for (i = 0; i < sequencesSize; ++i) { + assert(sequences[i].offset == expectedOffsets[i]); + assert(sequences[i].litLength == expectedLitLengths[i]); + assert(sequences[i].matchLength == expectedMatchLengths[i]); + assert(sequences[i].rep == expectedReps[i]); + assert(sequences[i].matchPos == expectedMatchPos[i]); + } ZSTD_freeCCtx(cctx); } From c04245b257e25c26cdb15507308575b1f16108e4 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 23 Sep 2019 15:42:16 -0700 Subject: [PATCH 022/106] Replacing assert with memory_allocation error code throw --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8eb292894..7facbeff0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2323,7 +2323,7 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); SeqCollector seqCollector; - assert(dst != NULL); + RETURN_ERROR_IF(dst == NULL, memory_allocation); seqCollector.collectSequences = 1; seqCollector.seqStart = outSeqs; From cb18fffe6544eda6e73b53dd9870e7393a267b3c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 24 Sep 2019 17:50:58 -0700 Subject: [PATCH 023/106] enforce C90 compatibility for zlibWrapper --- lib/common/zstd_internal.h | 2 +- lib/zstd.h | 10 +++++----- zlibWrapper/Makefile | 2 +- zlibWrapper/examples/fitblk.c | 8 ++++---- zlibWrapper/gzclose.c | 2 +- zlibWrapper/gzlib.c | 16 ++++++++-------- zlibWrapper/gzread.c | 16 ++++++++-------- zlibWrapper/gzwrite.c | 16 ++++++++-------- zlibWrapper/zstd_zlibwrapper.c | 16 ++++++++++------ 9 files changed, 46 insertions(+), 42 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index f791c5b38..d292f8a71 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -202,7 +202,7 @@ static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } typedef enum { ZSTD_no_overlap, - ZSTD_overlap_src_before_dst, + ZSTD_overlap_src_before_dst /* ZSTD_overlap_dst_before_src, */ } ZSTD_overlap_e; diff --git a/lib/zstd.h b/lib/zstd.h index 42d4188c8..08c851b22 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -398,7 +398,7 @@ typedef enum { ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, ZSTD_c_experimentalParam6=1003, - ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam7=1004 } ZSTD_cParameter; typedef struct { @@ -1106,7 +1106,7 @@ typedef enum { typedef enum { ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ - ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ } ZSTD_dictLoadMethod_e; typedef enum { @@ -1120,7 +1120,7 @@ typedef enum { * This question could be kept for later, when there are actually multiple formats to support, * but there is also the question of pinning enum values, and pinning value `0` is especially important */ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ - ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number. + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. * Useful to save 4 bytes per generated frame. * Decoder cannot recognise automatically this format, requiring this instruction. */ } ZSTD_format_e; @@ -1154,7 +1154,7 @@ typedef enum { */ ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ - ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceCopy = 2 /* Always copy the dictionary. */ } ZSTD_dictAttachPref_e; typedef enum { @@ -1163,7 +1163,7 @@ typedef enum { * levels will be compressed. */ ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be * emitted if Huffman compression is not profitable. */ - ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ } ZSTD_literalCompressionMode_e; diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index d4fc33b51..9222141ae 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -20,7 +20,7 @@ TEST_FILE = ../doc/zstd_compression_format.md CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99 +CFLAGS ?= $(MOREFLAGS) -O3 -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ -Wstrict-aliasing=1 diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 6418ca387..6b587ceeb 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -180,8 +180,8 @@ int main(int argc, char **argv) if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { /* write block to stdout */ have = size + EXCESS - def.avail_out; - // if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) - // quit("error writing output"); + /* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + * quit("error writing output"); */ /* clean up and print results to stderr */ ret = deflateEnd(&def); @@ -237,8 +237,8 @@ int main(int argc, char **argv) /* done -- write block to stdout */ have = size - def.avail_out; -// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) -// quit("error writing output"); + /* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + * quit("error writing output"); */ /* clean up and print results to stderr */ free(tmp); diff --git a/zlibWrapper/gzclose.c b/zlibWrapper/gzclose.c index d4493d010..25d3789b1 100644 --- a/zlibWrapper/gzclose.c +++ b/zlibWrapper/gzclose.c @@ -19,7 +19,7 @@ int ZEXPORT gzclose(file) if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; return state.state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); #else diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c index 3070dd8b4..b004e4f30 100644 --- a/zlibWrapper/gzlib.c +++ b/zlibWrapper/gzlib.c @@ -325,7 +325,7 @@ int ZEXPORT gzbuffer(file, size) /* get internal structure and check integrity */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return -1; @@ -351,7 +351,7 @@ int ZEXPORT gzrewind(file) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no error */ if (state.state->mode != GZ_READ || @@ -378,7 +378,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence) /* get internal structure and check integrity */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return -1; @@ -463,7 +463,7 @@ z_off64_t ZEXPORT gztell64(file) /* get internal structure and check integrity */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return -1; @@ -491,7 +491,7 @@ z_off64_t ZEXPORT gzoffset64(file) /* get internal structure and check integrity */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return -1; @@ -523,7 +523,7 @@ int ZEXPORT gzeof(file) /* get internal structure and check integrity */ if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return 0; @@ -541,7 +541,7 @@ const char * ZEXPORT gzerror(file, errnum) /* get internal structure and check integrity */ if (file == NULL) return NULL; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return NULL; @@ -561,7 +561,7 @@ void ZEXPORT gzclearerr(file) /* get internal structure and check integrity */ if (file == NULL) return; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return; diff --git a/zlibWrapper/gzread.c b/zlibWrapper/gzread.c index 88fc06c77..bcac9700d 100644 --- a/zlibWrapper/gzread.c +++ b/zlibWrapper/gzread.c @@ -386,7 +386,7 @@ int ZEXPORT gzread(file, buf, len) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -424,7 +424,7 @@ z_size_t ZEXPORT gzfread(buf, size, nitems, file) /* get internal structure */ if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -470,7 +470,7 @@ int ZEXPORT gzgetc(file) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -485,7 +485,7 @@ int ZEXPORT gzgetc(file) } /* nothing there -- try gz_read() */ - ret = (unsigned)gz_read(state, buf, 1); + ret = (int)gz_read(state, buf, 1); return ret < 1 ? -1 : buf[0]; } @@ -505,7 +505,7 @@ int ZEXPORT gzungetc(c, file) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -569,7 +569,7 @@ char * ZEXPORT gzgets(file, buf, len) /* check parameters and get internal structure */ if (file == NULL || buf == NULL || len < 1) return NULL; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -628,7 +628,7 @@ int ZEXPORT gzdirect(file) /* get internal structure */ if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; /* if the state is not known, but we can find out, then do so (this is mainly for right after a gzopen() or gzdopen()) */ @@ -649,7 +649,7 @@ int ZEXPORT gzclose_r(file) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; /* check that we're reading */ if (state.state->mode != GZ_READ) diff --git a/zlibWrapper/gzwrite.c b/zlibWrapper/gzwrite.c index 21d5f8472..422ff17db 100644 --- a/zlibWrapper/gzwrite.c +++ b/zlibWrapper/gzwrite.c @@ -258,7 +258,7 @@ int ZEXPORT gzwrite(file, buf, len) /* get internal structure */ if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; /* check that we're writing and that there's no error */ if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) @@ -289,7 +289,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) assert(size != 0); if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; /* check that we're writing and that there's no error */ if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) @@ -319,7 +319,7 @@ int ZEXPORT gzputc(file, c) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; strm = &(state.state->strm); /* check that we're writing and that there's no error */ @@ -366,7 +366,7 @@ int ZEXPORT gzputs(file, str) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're writing and that there's no error */ if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) @@ -393,7 +393,7 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; strm = &(state.state->strm); /* check that we're writing and that there's no error */ @@ -565,7 +565,7 @@ int ZEXPORT gzflush(file, flush) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; /* check that we're writing and that there's no error */ if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) @@ -599,7 +599,7 @@ int ZEXPORT gzsetparams(file, level, strategy) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; strm = &(state.state->strm); /* check that we're writing and that there's no error */ @@ -639,7 +639,7 @@ int ZEXPORT gzclose_w(file) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; /* check that we're writing */ if (state.state->mode != GZ_WRITE) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 0ee5a3108..d07ed9538 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -54,7 +54,7 @@ int ZWRAP_isUsingZSTDcompression(void) { return g_ZWRAP_useZSTDcompression; } static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO; -void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; }; +void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; } ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompressionType; } @@ -121,8 +121,10 @@ static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) if (zwc==NULL) return NULL; memset(zwc, 0, sizeof(ZWRAP_CCtx)); memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); - { ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc }; - zwc->customMem = ZWRAP_customMem; } + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL }; + ZWRAP_customMem.opaque = &zwc->allocFunc; + zwc->customMem = ZWRAP_customMem; + } } else { zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc)); if (zwc==NULL) return NULL; @@ -462,8 +464,10 @@ static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) if (zwd==NULL) return NULL; memset(zwd, 0, sizeof(ZWRAP_DCtx)); zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */ - { ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc }; - zwd->customMem = ZWRAP_customMem; } + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL }; + ZWRAP_customMem.opaque = &zwd->allocFunc; + zwd->customMem = ZWRAP_customMem; + } } else { zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd)); if (zwd==NULL) return NULL; @@ -1003,7 +1007,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm)) } -ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); }; +ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); } From 0582b27caeda307eddf8965bfc33f863b716cd3d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Sep 2019 13:08:25 -0700 Subject: [PATCH 024/106] added c++-compat build flag ensure code can be compiled "as is" in C++ mode also : restructured flags so that they can be individually changed / disabled on command line --- zlibWrapper/Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 9222141ae..a61980c23 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -18,12 +18,14 @@ EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs TEST_FILE = ../doc/zstd_compression_format.md -CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ +CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -CFLAGS ?= $(MOREFLAGS) -O3 -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ +STDFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat +DEBUGFLAGS=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ -Wstrict-aliasing=1 +CFLAGS ?= -O3 +CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) # Define *.exe as extension for Windows systems From 69c9401932e1b84af7381e2f5be73bf258225408 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Sep 2019 15:01:29 -0700 Subject: [PATCH 025/106] simplified Makefile and fixed a few c++-compat issues --- zlibWrapper/Makefile | 23 +++++++++++------------ zlibWrapper/examples/fitblk.c | 4 ++-- zlibWrapper/examples/zwrapbench.c | 24 ++++++++++++------------ zlibWrapper/zstd_zlibwrapper.c | 6 +++--- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index a61980c23..4ef180282 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -70,35 +70,34 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench $(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests #.c.o: -# $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< +# $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ -minigzip: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) +minigzip: $(EXAMPLE_PATH)/minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@ -minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) +minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@ -example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) +example: $(EXAMPLE_PATH)/example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) +example_zstd: $(EXAMPLE_PATH)/example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) +fitblk: $(EXAMPLE_PATH)/fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) +fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) +zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h - $(CC) $(CFLAGS) $(CPPFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c +zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h -$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h - $(CC) $(CFLAGS) $(CPPFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c +zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1 +zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h $(ZSTDLIBDIR)/libzstd.a: $(MAKE) -C $(ZSTDLIBDIR) libzstd.a diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 6b587ceeb..669b176eb 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -159,7 +159,7 @@ int main(int argc, char **argv) if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); /* allocate memory for buffers and compression engine */ - blk = malloc(size + EXCESS); + blk = (unsigned char*)malloc(size + EXCESS); def.zalloc = Z_NULL; def.zfree = Z_NULL; def.opaque = Z_NULL; @@ -200,7 +200,7 @@ int main(int argc, char **argv) inf.avail_in = 0; inf.next_in = Z_NULL; ret = inflateInit(&inf); - tmp = malloc(size + EXCESS); + tmp = (unsigned char*)malloc(size + EXCESS); if (ret != Z_OK || tmp == NULL) quit("out of memory"); ret = deflateReset(&def); diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 61031b9de..35893a4b4 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -311,14 +311,14 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, ret = deflateReset(&def); if (ret != Z_OK) EXM_THROW(1, "deflateReset failure"); if (useSetDict) { - ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize); + ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */ } - def.next_in = (z_const void*) blockTable[blockNb].srcPtr; + def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr; def.avail_in = (uInt)blockTable[blockNb].srcSize; def.total_in = 0; - def.next_out = (void*) blockTable[blockNb].cPtr; + def.next_out = (z_Bytef*) blockTable[blockNb].cPtr; def.avail_out = (uInt)blockTable[blockNb].cRoom; def.total_out = 0; ret = deflate(&def, Z_FINISH); @@ -343,13 +343,13 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, ret = deflateInit(&def, cLevel); if (ret != Z_OK) EXM_THROW(1, "deflateInit failure"); if (dictBuffer) { - ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize); + ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); } - def.next_in = (z_const void*) blockTable[blockNb].srcPtr; + def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr; def.avail_in = (uInt)blockTable[blockNb].srcSize; def.total_in = 0; - def.next_out = (void*) blockTable[blockNb].cPtr; + def.next_out = (z_Bytef*) blockTable[blockNb].cPtr; def.avail_out = (uInt)blockTable[blockNb].cRoom; def.total_out = 0; ret = deflate(&def, Z_FINISH); @@ -451,15 +451,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, else ret = inflateReset(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateReset failure"); - inf.next_in = (z_const void*) blockTable[blockNb].cPtr; + inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr; inf.avail_in = (uInt)blockTable[blockNb].cSize; inf.total_in = 0; - inf.next_out = (void*) blockTable[blockNb].resPtr; + inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr; inf.avail_out = (uInt)blockTable[blockNb].srcSize; inf.total_out = 0; ret = inflate(&inf, Z_FINISH); if (ret == Z_NEED_DICT) { - ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize); + ret = inflateSetDictionary(&inf, (const z_Bytef*)dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure"); ret = inflate(&inf, Z_FINISH); } @@ -483,15 +483,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, inf.opaque = Z_NULL; ret = inflateInit(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateInit failure"); - inf.next_in = (z_const void*) blockTable[blockNb].cPtr; + inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr; inf.avail_in = (uInt)blockTable[blockNb].cSize; inf.total_in = 0; - inf.next_out = (void*) blockTable[blockNb].resPtr; + inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr; inf.avail_out = (uInt)blockTable[blockNb].srcSize; inf.total_out = 0; ret = inflate(&inf, Z_FINISH); if (ret == Z_NEED_DICT) { - ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize); + ret = inflateSetDictionary(&inf, (const z_Bytef*) dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure"); ret = inflate(&inf, Z_FINISH); } diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index d07ed9538..cb6afa786 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -99,7 +99,7 @@ typedef struct { unsigned long long pledgedSrcSize; } ZWRAP_CCtx; -typedef ZWRAP_CCtx internal_state; +/* typedef ZWRAP_CCtx internal_state; */ @@ -513,7 +513,7 @@ static int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message) ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, - const char *version, int stream_size)) + const char* version, int stream_size)) { if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) { strm->reserved = ZWRAP_ZLIB_STREAM; @@ -524,7 +524,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, LOG_WRAPPERD("- inflateInit\n"); if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); - zwd->version = ZSTD_malloc(strlen(version)+1, zwd->customMem); + zwd->version = (char*)ZSTD_malloc(strlen(version)+1, zwd->customMem); if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); strcpy(zwd->version, version); From bb27472afcf5a8f6499638f30221e62850bba0e7 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 26 Sep 2019 15:38:31 -0700 Subject: [PATCH 026/106] Adding more realistic test for get sequences --- tests/fuzzer.c | 63 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 349f0d19b..66d1c7e03 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -304,6 +304,26 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) #endif +static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, BYTE* src, size_t size) +{ + size_t i; + for(i = 0; i < seqsSize; ++i) { + assert(dst + seqs[i].litLength + seqs[i].matchLength < dst + size); + assert(src + seqs[i].litLength + seqs[i].matchLength < src + size); + + memcpy(dst, src, seqs[i].litLength); + dst += seqs[i].litLength; + src += seqs[i].litLength; + size -= seqs[i].litLength; + + memcpy(dst, dst-seqs[i].offset, seqs[i].matchLength); + dst += seqs[i].matchLength; + src += seqs[i].matchLength; + size -= seqs[i].matchLength; + } + memcpy(dst, src, size); +} + /*============================================= * Unit tests =============================================*/ @@ -1960,32 +1980,31 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "OK \n"); } + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences decode from sequences test : ", testNb++); { + size_t srcSize = sizeof(U32) * 1000; + BYTE* src = (BYTE*)CNBuffer; + BYTE* decoded = (BYTE*)compressedBuffer; + ZSTD_CCtx* cctx = ZSTD_createCCtx(); - size_t zerosLength = ZSTD_BLOCKSIZE_MAX * 2 - 1; - size_t expectedOffsets[] = {1, 1}; - size_t expectedLitLengths[] = {2, 1}; - size_t expectedMatchLengths[] = {ZSTD_BLOCKSIZE_MAX - 2, ZSTD_BLOCKSIZE_MAX - 2}; - size_t expectedReps[] = {1, 1}; - size_t expectedMatchPos[] = {2, 1}; - size_t expectedSequencesSize = 2; - size_t sequencesSize; - size_t i = 0; - ZSTD_Sequence* sequences = (ZSTD_Sequence*)compressedBuffer; - DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences zeros : ", testNb++); + ZSTD_Sequence* seqs = malloc(srcSize * sizeof(ZSTD_Sequence)); + size_t seqsSize; size_t i; + U32 randSeed = seed; + assert(cctx != NULL); - memset(CNBuffer, 0, zerosLength); - sequencesSize = ZSTD_getSequences(cctx, sequences, 10, - CNBuffer, zerosLength); - assert(sequencesSize == expectedSequencesSize); - for (i = 0; i < sequencesSize; ++i) { - assert(sequences[i].offset == expectedOffsets[i]); - assert(sequences[i].litLength == expectedLitLengths[i]); - assert(sequences[i].matchLength == expectedMatchLengths[i]); - assert(sequences[i].rep == expectedReps[i]); - assert(sequences[i].matchPos == expectedMatchPos[i]); - } + + /* Populate src with random data */ + for (i = 0; i < srcSize / sizeof(U32); ++i) {*((U32*)src + i) = FUZ_rand(&randSeed);} + + /* get the sequences */ + seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize); + + /* "decode" and compare the sequences */ + FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize); + assert(!memcmp(CNBuffer, compressedBuffer, 5)); + ZSTD_freeCCtx(cctx); + free(seqs); } /* Multiple blocks of zeros test */ From 75b128635455994ea5218036c5bbd187c67e1998 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 26 Sep 2019 16:07:34 -0700 Subject: [PATCH 027/106] Fixing shortest failure --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 66d1c7e03..c36879747 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1994,7 +1994,7 @@ static int basicUnitTests(U32 const seed, double compressibility) assert(cctx != NULL); /* Populate src with random data */ - for (i = 0; i < srcSize / sizeof(U32); ++i) {*((U32*)src + i) = FUZ_rand(&randSeed);} + for (i = 0; i < srcSize / sizeof(U32); ++i) {((U32*)CNBuffer)[i] = FUZ_rand(&randSeed);} /* get the sequences */ seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize); From 91daee5c068c105d456331ff30fd6971ec9cd437 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 26 Sep 2019 16:21:57 -0700 Subject: [PATCH 028/106] Fixing appveyor test --- tests/fuzzer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index c36879747..0b935c825 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1987,10 +1987,11 @@ static int basicUnitTests(U32 const seed, double compressibility) BYTE* decoded = (BYTE*)compressedBuffer; ZSTD_CCtx* cctx = ZSTD_createCCtx(); - ZSTD_Sequence* seqs = malloc(srcSize * sizeof(ZSTD_Sequence)); + ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); size_t seqsSize; size_t i; U32 randSeed = seed; + if (seqs == NULL) goto _output_error; assert(cctx != NULL); /* Populate src with random data */ From b63a1e7ae526e7544c0bcfdfebf8744f133eb500 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 27 Sep 2019 07:20:20 -0700 Subject: [PATCH 029/106] Typo fix --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 0b935c825..cf81230db 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -2002,7 +2002,7 @@ static int basicUnitTests(U32 const seed, double compressibility) /* "decode" and compare the sequences */ FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize); - assert(!memcmp(CNBuffer, compressedBuffer, 5)); + assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); ZSTD_freeCCtx(cctx); free(seqs); From f80437c58665294083c27ebba519b303352fa437 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 11:08:20 -0400 Subject: [PATCH 030/106] Add support for --output-dir-flat New flag to specify output directory destination for multiple files. --- build/LICENSE | 0 programs/fileio.c | 2 +- programs/util.c | 94 ++++++++++++++++++++++++++++++---------------- programs/util.h | 11 +++--- programs/zstdcli.c | 27 ++++++++----- tests/playTests.sh | 19 +++++++++- 6 files changed, 103 insertions(+), 50 deletions(-) create mode 100644 build/LICENSE diff --git a/build/LICENSE b/build/LICENSE new file mode 100644 index 000000000..e69de29bb diff --git a/programs/fileio.c b/programs/fileio.c index cc8809f94..c0300f6a0 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1453,7 +1453,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - if (outDirName) + if (dstFileNamesTable) UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; diff --git a/programs/util.c b/programs/util.c index 920601fba..e5c937864 100644 --- a/programs/util.c +++ b/programs/util.c @@ -90,53 +90,81 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } -int UTIL_createDir(const char* outDirName) -{ - int r; - if (UTIL_isDirectory(outDirName)) - return 0; /* no need to create if directory already exists */ +int UTIL_compareStr(const void *p1, const void *p2) { + return strcmp(* (char * const *) p1, * (char * const *) p2); +} -#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) - r = _mkdir(outDirName); - if (r || !UTIL_isDirectory(outDirName)) return 1; -#else - r = mkdir(outDirName, S_IRWXU | S_IRWXG | S_IRWXO); /* dir has all permissions */ - if (r || !UTIL_isDirectory(outDirName)) return 1; -#endif +int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles) { + char** dstFilenameTableSorted; + char* prevElem; + unsigned u; + + dstFilenameTableSorted = (char**) malloc(sizeof(char*) * nbFiles); + if (!dstFilenameTableSorted) { + UTIL_DISPLAYLEVEL(1, "Unable to malloc new str array, not checking for name collisions\n"); + return 1; + } + + for (u = 0; u < nbFiles; ++u) { + dstFilenameTableSorted[u] = dstFilenameTable[u]; + } + qsort(dstFilenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + prevElem = dstFilenameTableSorted[0]; + for (u = 1; u < nbFiles; ++u) { + if (strcmp(prevElem, dstFilenameTableSorted[u]) == 0) { + UTIL_DISPLAYLEVEL(1, "WARNING: Two files have same filename as source : %s\n", prevElem); + } + prevElem = dstFilenameTableSorted[u]; + } + + free(dstFilenameTableSorted); return 0; } -void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles, - const char* outDirName, char** dstFilenameTable) +void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, + const unsigned nbFiles, const char* outDirName, const int compressing) { unsigned u; - char c; - c = '/'; + const char* c; + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + c = "\\"; + #else + c = "/"; + #endif - /* duplicate source file table */ for (u = 0; u < nbFiles; ++u) { - const char* filename; + char* filename, *filenameBegin; size_t finalPathLen; finalPathLen = strlen(outDirName); - filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ + filenameBegin = strrchr(filenameTable[u], c[0]); + if (filenameBegin == NULL) { + filename = strdup(filenameTable[u]); + } else { + filename = strdup(filenameBegin+1); + } + finalPathLen += strlen(filename); - dstFilenameTable[u] = (char*) malloc((finalPathLen+5) * sizeof(char)); /* extra 1 bit for \0, extra 4 for .zst if compressing*/ + dstFilenameTable[u] = compressing ? + (char*) malloc((finalPathLen+6) * sizeof(char)) /* 4 more bytes for .zst suffix */ + : (char*) malloc((finalPathLen+2) * sizeof(char)); + if (!dstFilenameTable[u]) { + UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n"); + continue; + } + strcpy(dstFilenameTable[u], outDirName); - strcat(dstFilenameTable[u], filename); - } -} + if (outDirName[strlen(outDirName)-1] == c[0]) { + strcat(dstFilenameTable[u], filename); + } else { + strcat(dstFilenameTable[u], c); + strcat(dstFilenameTable[u], filename); + } -void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName) { - int dirResult; - dirResult = UTIL_createDir(outDirName); - if (dirResult) - UTIL_DISPLAYLEVEL(1, "Directory creation unsuccessful\n"); + free(filename); + } - UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); - if (outFileName) { - outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + if (UTIL_checkFilenameCollisions(dstFilenameTable, nbFiles)) { + UTIL_DISPLAYLEVEL(1, "Checking for filename collisions failed"); } } diff --git a/programs/util.h b/programs/util.h index 9615504ca..e90b251de 100644 --- a/programs/util.h +++ b/programs/util.h @@ -127,15 +127,14 @@ int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); int UTIL_setFileStat(const char* filename, stat_t* statbuf); U32 UTIL_isDirectory(const char* infilename); -int UTIL_createDir(const char* outDirName); int UTIL_getFileStat(const char* infilename, stat_t* statbuf); int UTIL_isSameFile(const char* file1, const char* file2); -void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx, - const char* outDirName, char** dstFilenameTable); +int UTIL_compareStr(const void *p1, const void *p2); +int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles); +/* Populates dstFilenameTable using outDirName concatenated with entries from filenameTable */ +void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, const unsigned nbFiles, + const char* outDirName, const int compressing); void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); -void UTIL_processMultipleFilenameDestinationDir(char** dstFilenameTable, - const char** filenameTable, unsigned filenameIdx, - const char* outFileName, const char* outDirName); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 5fc6b8aa8..6b761ffa6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -118,7 +118,6 @@ static int usage(const char* programName) #endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); - DISPLAY( " -O directory: result(s) stored into `directory`, creates one if non-existent \n"); DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); @@ -137,6 +136,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -l : print information about zstd compressed files \n"); + DISPLAY( " --output-dir-flat directory: results stored into `directory` top level \n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); @@ -690,6 +690,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; } if (!strcmp(argument, "--adapt")) { adapt = 1; continue; } if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; } if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } @@ -856,9 +857,6 @@ int main(int argCount, const char* argv[]) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; - - /* destination directory name */ - case 'O': nextArgumentIsOutDirName=1; lastCommand=1; argument++; break; /* limit decompression memory */ case 'M': @@ -1178,9 +1176,14 @@ int main(int argCount, const char* argv[]) if (adaptMax < cLevel) cLevel = adaptMax; if (outDirName) { - printf("ok\n"); - dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + if (UTIL_isDirectory(outDirName)) { + DISPLAY("Output of files will be in directory: %s\n", outDirName); + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); + UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1); + } else { + DISPLAY("%s is not a directory!\n", outDirName); + CLEAN_RETURN(1); + } } else { dstFilenameTable = NULL; } @@ -1205,8 +1208,14 @@ int main(int argCount, const char* argv[]) FIO_setMemLimit(prefs, memLimit); if (outDirName) { - dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_processMultipleFilenameDestinationDir(dstFilenameTable, filenameTable, filenameIdx, outFileName, outDirName); + if (UTIL_isDirectory(outDirName)) { + DISPLAY("Output of files will be in directory: %s\n", outDirName); + dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); + UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1); + } else { + DISPLAY("%s is not a directory!\n", outDirName); + CLEAN_RETURN(1); + } } else { dstFilenameTable = NULL; } diff --git a/tests/playTests.sh b/tests/playTests.sh index 19fc514f6..8851da2f5 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -264,6 +264,24 @@ if [ "$?" -eq 139 ]; then fi rm tmp* +println "test : compress multiple files into an output directory, --output-dir-flat" +println henlo > tmp1 +mkdir tmpInputTestDir +mkdir tmpInputTestDir/we +mkdir tmpInputTestDir/we/must +mkdir tmpInputTestDir/we/must/go +mkdir tmpInputTestDir/we/must/go/deeper +println cool > tmpInputTestDir/we/must/go/deeper/tmp2 +mkdir tmpOutDir +$ZSTD tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir +test -f tmpOutDir/tmp1.zst +test -f tmpOutDir/tmp2.zst +println "test : decompress multiple files into an output directory, --output-dir-flat" +mkdir tmpOutDirDecomp +$ZSTD tmpOutDir/ -r -d --output-dir-flat tmpOutDirDecomp +test -f tmpOutDirDecomp/tmp2 +test -f tmpOutDirDecomp/tmp1 +rm -rf tmp* println "\n===> Advanced compression parameters " println "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" @@ -407,7 +425,6 @@ ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) println "compress multiple files including a missing one (notHere) : " $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" - println "\n===> stream-size mode" ./datagen -g11000 > tmp From 1e4d602412241c142b7f57e1d7f4995e12a003e0 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 11:11:35 -0400 Subject: [PATCH 031/106] Modified message --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6b761ffa6..63e71e7e6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -136,7 +136,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); DISPLAY( " -c : force write to standard output, even if it is the console\n"); DISPLAY( " -l : print information about zstd compressed files \n"); - DISPLAY( " --output-dir-flat directory: results stored into `directory` top level \n"); + DISPLAY( " --output-dir-flat directory: results stored into `directory`. Filename collisions mean first file will be compressed. With -f, the last file will be compressed.\n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog); From 147b7614544e411e2087c3e696f4c1be046986cf Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 11:18:14 -0400 Subject: [PATCH 032/106] Removed strdup() dependency --- programs/util.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/util.c b/programs/util.c index e5c937864..083eb12e0 100644 --- a/programs/util.c +++ b/programs/util.c @@ -138,9 +138,11 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena finalPathLen = strlen(outDirName); filenameBegin = strrchr(filenameTable[u], c[0]); if (filenameBegin == NULL) { - filename = strdup(filenameTable[u]); + filename = (char*) malloc((strlen(filenameTable[u])+1) * sizeof(char)); + strcpy(filename, filenameTable[u]); } else { - filename = strdup(filenameBegin+1); + filename = (char*) malloc((strlen(filenameBegin+1)) * sizeof(char)); + strcpy(filename, filenameBegin+1); } finalPathLen += strlen(filename); From b93f1b2a30c55cc0d5938c71f33b9e83f28d2d7a Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 11:29:34 -0400 Subject: [PATCH 033/106] CI Tests fix --- programs/util.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/util.c b/programs/util.c index 083eb12e0..aa769567e 100644 --- a/programs/util.c +++ b/programs/util.c @@ -133,7 +133,8 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena #endif for (u = 0; u < nbFiles; ++u) { - char* filename, *filenameBegin; + char* filename; + const char* filenameBegin; size_t finalPathLen; finalPathLen = strlen(outDirName); filenameBegin = strrchr(filenameTable[u], c[0]); From c763457e0a4d70b6912bc14c5908a38c3239a0bf Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 15:30:24 -0400 Subject: [PATCH 034/106] Static analyze fix --- programs/util.c | 7 +++---- programs/util.h | 2 +- programs/zstdcli.c | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/programs/util.c b/programs/util.c index aa769567e..83fa70bae 100644 --- a/programs/util.c +++ b/programs/util.c @@ -122,7 +122,7 @@ int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles) { } void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, - const unsigned nbFiles, const char* outDirName, const int compressing) + const unsigned nbFiles, const char* outDirName) { unsigned u; const char* c; @@ -147,11 +147,10 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena } finalPathLen += strlen(filename); - dstFilenameTable[u] = compressing ? - (char*) malloc((finalPathLen+6) * sizeof(char)) /* 4 more bytes for .zst suffix */ - : (char*) malloc((finalPathLen+2) * sizeof(char)); + dstFilenameTable[u] = (char*) malloc((finalPathLen+2) * sizeof(char)); if (!dstFilenameTable[u]) { UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n"); + free(filename); continue; } diff --git a/programs/util.h b/programs/util.h index e90b251de..558e7a9c6 100644 --- a/programs/util.h +++ b/programs/util.h @@ -133,7 +133,7 @@ int UTIL_compareStr(const void *p1, const void *p2); int UTIL_checkFilenameCollisions(char** dstFilenameTable, unsigned nbFiles); /* Populates dstFilenameTable using outDirName concatenated with entries from filenameTable */ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filenameTable, const unsigned nbFiles, - const char* outDirName, const int compressing); + const char* outDirName); void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); U32 UTIL_isLink(const char* infilename); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 63e71e7e6..d9e1be66a 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1179,7 +1179,7 @@ int main(int argCount, const char* argv[]) if (UTIL_isDirectory(outDirName)) { DISPLAY("Output of files will be in directory: %s\n", outDirName); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1); + UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName); } else { DISPLAY("%s is not a directory!\n", outDirName); CLEAN_RETURN(1); @@ -1211,7 +1211,7 @@ int main(int argCount, const char* argv[]) if (UTIL_isDirectory(outDirName)) { DISPLAY("Output of files will be in directory: %s\n", outDirName); dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName, 1); + UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName); } else { DISPLAY("%s is not a directory!\n", outDirName); CLEAN_RETURN(1); From 4dc604cab8f5d5e89454d2afeffe32f2001c41aa Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 2 Oct 2019 18:34:42 -0400 Subject: [PATCH 035/106] Addressed comments on malloc --- programs/fileio.c | 6 ------ programs/util.c | 12 ++++++++++-- programs/zstdcli.c | 5 +++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c0300f6a0..6006af01b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1453,9 +1453,6 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN } } FIO_freeCResources(ress); - if (dstFileNamesTable) - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); - return error; } @@ -2276,9 +2273,6 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } FIO_freeDResources(ress); - if (outDirName) - UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); - return error; } diff --git a/programs/util.c b/programs/util.c index 83fa70bae..c12a31887 100644 --- a/programs/util.c +++ b/programs/util.c @@ -140,14 +140,22 @@ void UTIL_createDestinationDirTable(char** dstFilenameTable, const char** filena filenameBegin = strrchr(filenameTable[u], c[0]); if (filenameBegin == NULL) { filename = (char*) malloc((strlen(filenameTable[u])+1) * sizeof(char)); + if (!filename) { + UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n"); + continue; + } strcpy(filename, filenameTable[u]); } else { - filename = (char*) malloc((strlen(filenameBegin+1)) * sizeof(char)); + filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char)); + if (!filename) { + UTIL_DISPLAYLEVEL(1, "Unable to allocate space for filename str\n"); + continue; + } strcpy(filename, filenameBegin+1); } finalPathLen += strlen(filename); - dstFilenameTable[u] = (char*) malloc((finalPathLen+2) * sizeof(char)); + dstFilenameTable[u] = (char*) malloc((finalPathLen+3) * sizeof(char)); if (!dstFilenameTable[u]) { UTIL_DISPLAYLEVEL(1, "Unable to allocate space for file destination str\n"); free(filename); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d9e1be66a..91e67a2d3 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1192,6 +1192,9 @@ int main(int argCount, const char* argv[]) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); + + if (dstFilenameTable) + UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx); #else (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); @@ -1224,6 +1227,8 @@ int main(int argCount, const char* argv[]) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName); + if (dstFilenameTable) + UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx); #else DISPLAY("Decompression not supported \n"); #endif From c69ed0f8d7a2db6b2ebb287ce28beff109af75ef Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Oct 2019 17:32:19 -0700 Subject: [PATCH 036/106] updated tests for zlibwrapper C90 strict compatibility --- zlibWrapper/Makefile | 23 ++++++++++++++--------- zlibWrapper/gzlib.c | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 4ef180282..f8ade8282 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -18,15 +18,15 @@ EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs TEST_FILE = ../doc/zstd_compression_format.md -CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ - -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -STDFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat -DEBUGFLAGS=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ - -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ - -Wstrict-aliasing=1 -CFLAGS ?= -O3 -CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) - +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ + -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) +STDCFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \ + -DNO_snprintf -DNO_vsnprintf # strict ISO C90 is missing these prototypes +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ + -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ + -Wstrict-aliasing=1 +CFLAGS ?= -O3 +CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -35,6 +35,11 @@ else EXT = endif +default : release + +release : STDFLAGS = +release : STDCPPFLAGS = +release : all all: fitblk example zwrapbench minigzip diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c index b004e4f30..b1fb98517 100644 --- a/zlibWrapper/gzlib.c +++ b/zlibWrapper/gzlib.c @@ -216,7 +216,7 @@ local gzFile gz_open(path, fd, mode) #if !defined(NO_snprintf) && !defined(NO_vsnprintf) (void)snprintf(state.state->path, len + 1, "%s", (const char *)path); #else - strcpy(state.state->path, path); + strcpy(state.state->path, (const char*)path); #endif /* compute the flags for open() */ From cadff8cdc4c339aa0a85ef505c5a4a2130ff96af Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Oct 2019 17:41:19 -0700 Subject: [PATCH 037/106] zlibwrapper : fixed flag-variable name release doesn't use specific std/debug flags --- zlibWrapper/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index f8ade8282..f63291c90 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -20,7 +20,7 @@ TEST_FILE = ../doc/zstd_compression_format.md CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -STDCFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \ +STDFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \ -DNO_snprintf -DNO_vsnprintf # strict ISO C90 is missing these prototypes DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ @@ -38,7 +38,7 @@ endif default : release release : STDFLAGS = -release : STDCPPFLAGS = +release : DEBUGFLAGS = release : all all: fitblk example zwrapbench minigzip From 61ec4c2e7f7a20856933978b79dcf55a3c68ff74 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 3 Oct 2019 06:42:40 -0700 Subject: [PATCH 038/106] Cleaning sequence parsing logic --- lib/compress/zstd_compress.c | 59 +++++++++++++++--------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7facbeff0..4e2fbd853 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2272,46 +2272,37 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) size_t seqsSize = seqStore->sequences - seqs; ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; - size_t i; size_t position; int repIdx; + int i; assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); - for (i = 0, position = 0; i < seqsSize; ++i) { - outSeqs[i].offset = seqs[i].offset; + for (i = 0; i < (int)seqsSize; ++i) { + unsigned int offsetValue = seqs[i].offset; outSeqs[i].litLength = seqs[i].litLength; - outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH; - - if (i == seqStore->longLengthPos) { - if (seqStore->longLengthID == 1) { - outSeqs[i].litLength += 0x10000; - } else if (seqStore->longLengthID == 2) { - outSeqs[i].matchLength += 0x10000; - } - } - - if (outSeqs[i].offset <= ZSTD_REP_NUM) { - outSeqs[i].rep = outSeqs[i].offset; - repIdx = (unsigned int)i - outSeqs[i].offset; - - if (outSeqs[i].litLength == 0) { - if (outSeqs[i].offset < 3) { - --repIdx; - } else { - repIdx = (unsigned int)i - 1; - } - ++outSeqs[i].rep; - } - assert(repIdx >= -3); - outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; - if (outSeqs[i].rep == 4) { - --outSeqs[i].offset; - } + outSeqs[i].matchLength = seqs[i].matchLength; + if (offsetValue > 3) { + outSeqs[i].offset = offsetValue - 3; } else { - outSeqs[i].offset -= ZSTD_REP_NUM; + /* special repeat offset case */ + unsigned int repeatOffset1 = i - 1 >= 0 ? outSeqs[i - 1].offset : 1; + unsigned int repeatOffset2 = 1 - 2 >= 0 ? outSeqs[i - 2].offset : 4; + unsigned int repeatOffset3 = i - 3 >= 0 ? outSeqs[i - 3].offset : 8; + if (seqs[i].litLength != 0) { + switch (offsetValue) { + case 1: outSeqs[i].offset = repeatOffset1; break; + case 2: outSeqs[i].offset = repeatOffset2; break; + case 3: outSeqs[i].offset = repeatOffset3; break; + } + } else { + /* offsets shifted by one */ + switch (offsetValue) { + case 1: outSeqs[i].offset = repeatOffset2; break; + case 2: outSeqs[i].offset = repeatOffset3; break; + /* corner case where offsetValue == 3 */ + case 3: outSeqs[i].offset = repeatOffset1 - 1; break; + } + } } - position += outSeqs[i].litLength; - outSeqs[i].matchPos = (unsigned int)position; - position += outSeqs[i].matchLength; } zc->seqCollector.seqIndex += seqsSize; } From 36528b96c4ae2dff700e63851f710aa11c1d0bc8 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 3 Oct 2019 09:26:51 -0700 Subject: [PATCH 039/106] Manually moving instead of memcpy on decoder and using genBuffer() --- lib/compress/zstd_compress.c | 59 +++++++++++++++++++++--------------- tests/fuzzer.c | 13 ++++---- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4e2fbd853..7facbeff0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2272,37 +2272,46 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) size_t seqsSize = seqStore->sequences - seqs; ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; - int i; + size_t i; size_t position; int repIdx; assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); - for (i = 0; i < (int)seqsSize; ++i) { - unsigned int offsetValue = seqs[i].offset; + for (i = 0, position = 0; i < seqsSize; ++i) { + outSeqs[i].offset = seqs[i].offset; outSeqs[i].litLength = seqs[i].litLength; - outSeqs[i].matchLength = seqs[i].matchLength; - if (offsetValue > 3) { - outSeqs[i].offset = offsetValue - 3; - } else { - /* special repeat offset case */ - unsigned int repeatOffset1 = i - 1 >= 0 ? outSeqs[i - 1].offset : 1; - unsigned int repeatOffset2 = 1 - 2 >= 0 ? outSeqs[i - 2].offset : 4; - unsigned int repeatOffset3 = i - 3 >= 0 ? outSeqs[i - 3].offset : 8; - if (seqs[i].litLength != 0) { - switch (offsetValue) { - case 1: outSeqs[i].offset = repeatOffset1; break; - case 2: outSeqs[i].offset = repeatOffset2; break; - case 3: outSeqs[i].offset = repeatOffset3; break; - } - } else { - /* offsets shifted by one */ - switch (offsetValue) { - case 1: outSeqs[i].offset = repeatOffset2; break; - case 2: outSeqs[i].offset = repeatOffset3; break; - /* corner case where offsetValue == 3 */ - case 3: outSeqs[i].offset = repeatOffset1 - 1; break; - } + outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; } } + if (outSeqs[i].offset <= ZSTD_REP_NUM) { + outSeqs[i].rep = outSeqs[i].offset; + repIdx = (unsigned int)i - outSeqs[i].offset; + + if (outSeqs[i].litLength == 0) { + if (outSeqs[i].offset < 3) { + --repIdx; + } else { + repIdx = (unsigned int)i - 1; + } + ++outSeqs[i].rep; + } + assert(repIdx >= -3); + outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; + if (outSeqs[i].rep == 4) { + --outSeqs[i].offset; + } + } else { + outSeqs[i].offset -= ZSTD_REP_NUM; + } + + position += outSeqs[i].litLength; + outSeqs[i].matchPos = (unsigned int)position; + position += outSeqs[i].matchLength; } zc->seqCollector.seqIndex += seqsSize; } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index cf81230db..a22dafa36 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -307,7 +307,8 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, BYTE* src, size_t size) { size_t i; - for(i = 0; i < seqsSize; ++i) { + size_t j; + for(i = 0; i < seqsSize - 1; ++i) { assert(dst + seqs[i].litLength + seqs[i].matchLength < dst + size); assert(src + seqs[i].litLength + seqs[i].matchLength < src + size); @@ -316,7 +317,8 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, src += seqs[i].litLength; size -= seqs[i].litLength; - memcpy(dst, dst-seqs[i].offset, seqs[i].matchLength); + for (j = 0; j < seqs[i].matchLength; ++j) + dst[j] = dst[j - seqs[i].offset]; dst += seqs[i].matchLength; src += seqs[i].matchLength; size -= seqs[i].matchLength; @@ -1982,20 +1984,19 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences decode from sequences test : ", testNb++); { - size_t srcSize = sizeof(U32) * 1000; + size_t srcSize = 100 KB; BYTE* src = (BYTE*)CNBuffer; BYTE* decoded = (BYTE*)compressedBuffer; ZSTD_CCtx* cctx = ZSTD_createCCtx(); ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); - size_t seqsSize; size_t i; - U32 randSeed = seed; + size_t seqsSize; if (seqs == NULL) goto _output_error; assert(cctx != NULL); /* Populate src with random data */ - for (i = 0; i < srcSize / sizeof(U32); ++i) {((U32*)CNBuffer)[i] = FUZ_rand(&randSeed);} + RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); /* get the sequences */ seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize); From 64bc441d7d06b267aba52cfdae681a5582d4cfe7 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 3 Oct 2019 13:53:04 -0400 Subject: [PATCH 040/106] Now constructs final destination path without allocating new table --- programs/fileio.c | 184 +++++++++++++++++++++++++++++++++++---------- programs/fileio.h | 14 ++-- programs/util.c | 93 ----------------------- programs/util.h | 5 -- programs/zstdcli.c | 37 +-------- 5 files changed, 157 insertions(+), 176 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 6006af01b..9988e2d6a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -628,6 +628,104 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) return (size_t)fileSize; } + + +/* FIO_checkFilenameCollisions() : + * Checks for and warns if there are any files that would have the same output path + */ +int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { + const char** filenameTableSorted; + const char* c, *prevElem; + char* filename; + unsigned u; + + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + c = "\\"; + #else + c = "/"; + #endif + + filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles); + if (!filenameTableSorted) { + DISPLAY("Unable to malloc new str array, not checking for name collisions\n"); + return 1; + } + + for (u = 0; u < nbFiles; ++u) { + filename = strrchr(filenameTable[u], c[0]); + if (filename == NULL) { + filenameTableSorted[u] = filenameTable[u]; + } else { + filenameTableSorted[u] = filename+1; + } + } + + qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + prevElem = filenameTableSorted[0]; + for (u = 1; u < nbFiles; ++u) { + if (strcmp(prevElem, filenameTableSorted[u]) == 0) { + DISPLAY("WARNING: Two files have same filename: %s\n", prevElem); + } + prevElem = filenameTableSorted[u]; + } + + free(filenameTableSorted); + return 0; +} + +/* FIO_determineDstFilenameOutdir() : + * Takes a source file name and specified output directory, and + * allocates memory for and returns a pointer to final path. + * This function never returns an error (it may abort() in case of pb) + */ +static char* +FIO_determineDstFilenameOutdir(const char* srcFilename, const char* outDirName, const size_t suffixLen) +{ + const char* c, *filenameBegin; + char* filename, *result; + size_t finalPathLen; + + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + c = "\\"; + #else + c = "/"; + #endif + + finalPathLen = strlen(outDirName); + filenameBegin = strrchr(srcFilename, c[0]); + if (filenameBegin == NULL) { + filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char)); + if (!filename) { + EXM_THROW(30, "zstd: %s", strerror(errno)); + } + strcpy(filename, srcFilename); + } else { + filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char)); + if (!filename) { + EXM_THROW(30, "zstd: %s", strerror(errno)); + } + strcpy(filename, filenameBegin+1); + } + + finalPathLen += strlen(filename); + result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char)); + if (!result) { + free(filename); + EXM_THROW(30, "zstd: %s", strerror(errno)); + } + + strcpy(result, outDirName); + if (outDirName[strlen(outDirName)-1] == c[0]) { + strcat(result, filename); + } else { + strcat(result, c); + strcat(result, filename); + } + + free(filename); + return result; +} + #ifndef ZSTD_NOCOMPRESS /* ********************************************************************** @@ -1379,19 +1477,25 @@ int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName, return result; } - /* FIO_determineCompressedName() : * create a destination filename for compressed srcFileName. * @return a pointer to it. * This function never returns an error (it may abort() in case of pb) */ static const char* -FIO_determineCompressedName(const char* srcFileName, const char* suffix) +FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix) { static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - size_t const sfnSize = strlen(srcFileName); + char* outDirFilename = NULL; + size_t sfnSize = strlen(srcFileName); size_t const suffixSize = strlen(suffix); + if (outDirName) { + outDirFilename = FIO_determineDstFilenameOutdir(srcFileName, outDirName, suffixSize); + sfnSize = strlen(outDirFilename); + assert(outDirFilename != NULL); + } + if (dfnbCapacity <= sfnSize+suffixSize+1) { /* resize buffer for dstName */ free(dstFileNameBuffer); @@ -1399,9 +1503,16 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) dstFileNameBuffer = (char*)malloc(dfnbCapacity); if (!dstFileNameBuffer) { EXM_THROW(30, "zstd: %s", strerror(errno)); - } } + } + } assert(dstFileNameBuffer != NULL); - memcpy(dstFileNameBuffer, srcFileName, sfnSize); + + if (outDirFilename) { + memcpy(dstFileNameBuffer, outDirFilename, sfnSize); + free(outDirFilename); + } else { + memcpy(dstFileNameBuffer, srcFileName, sfnSize); + } memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */); return dstFileNameBuffer; } @@ -1414,24 +1525,17 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) * or into a destination folder (specified with -O) */ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, - const char* outDirName, char** dstFileNamesTable, - unsigned nbFiles, const char* outFileName, - const char* suffix, const char* dictFileName, - int compressionLevel, ZSTD_compressionParameters comprParams) + const char* outDirName, unsigned nbFiles, + const char* outFileName, const char* suffix, + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters comprParams) { int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); /* init */ assert(outFileName != NULL || suffix != NULL); - if (outDirName != NULL) { /* output into a particular folder */ - unsigned u; - for (u = 0; u < nbFiles; ++u) { - const char* const srcFileName = inFileNamesTable[u]; - const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix); - error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); - } - } else if (outFileName != NULL) { /* output into a single destination (stdout typically) */ + if (outFileName != NULL) { /* output into a single destination (stdout typically) */ ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; @@ -1448,9 +1552,11 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN unsigned u; for (u=0; u= 1 */ - char** dstFilenameTable; unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; @@ -1175,26 +1174,10 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; - if (outDirName) { - if (UTIL_isDirectory(outDirName)) { - DISPLAY("Output of files will be in directory: %s\n", outDirName); - dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName); - } else { - DISPLAY("%s is not a directory!\n", outDirName); - CLEAN_RETURN(1); - } - } else { - dstFilenameTable = NULL; - } - if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); - - if (dstFilenameTable) - UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx); + operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); @@ -1209,26 +1192,10 @@ int main(int argCount, const char* argv[]) } } FIO_setMemLimit(prefs, memLimit); - - if (outDirName) { - if (UTIL_isDirectory(outDirName)) { - DISPLAY("Output of files will be in directory: %s\n", outDirName); - dstFilenameTable = (char**)malloc(filenameIdx * sizeof(char*)); - UTIL_createDestinationDirTable(dstFilenameTable, filenameTable, filenameIdx, outDirName); - } else { - DISPLAY("%s is not a directory!\n", outDirName); - CLEAN_RETURN(1); - } - } else { - dstFilenameTable = NULL; - } - if (filenameIdx==1 && outFileName) operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName); else - operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, dstFilenameTable, outFileName, dictFileName); - if (dstFilenameTable) - UTIL_freeDestinationFilenameTable(dstFilenameTable, filenameIdx); + operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName); #else DISPLAY("Decompression not supported \n"); #endif From 6b81bfb257523f4638e0e1e6005d30776cdffb6e Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 3 Oct 2019 15:23:49 -0400 Subject: [PATCH 041/106] Changed function name, fix test --- programs/fileio.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9988e2d6a..6f1a2d678 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -634,9 +634,7 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) * Checks for and warns if there are any files that would have the same output path */ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { - const char** filenameTableSorted; - const char* c, *prevElem; - char* filename; + const char **filenameTableSorted, *c, *prevElem, *filename; unsigned u; #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ @@ -673,13 +671,13 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { return 0; } -/* FIO_determineDstFilenameOutdir() : +/* FIO_createFilename_fromOutDir() : * Takes a source file name and specified output directory, and * allocates memory for and returns a pointer to final path. * This function never returns an error (it may abort() in case of pb) */ static char* -FIO_determineDstFilenameOutdir(const char* srcFilename, const char* outDirName, const size_t suffixLen) +FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, const size_t suffixLen) { const char* c, *filenameBegin; char* filename, *result; @@ -1491,7 +1489,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con size_t sfnSize = strlen(srcFileName); size_t const suffixSize = strlen(suffix); if (outDirName) { - outDirFilename = FIO_determineDstFilenameOutdir(srcFileName, outDirName, suffixSize); + outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, suffixSize); sfnSize = strlen(outDirFilename); assert(outDirFilename != NULL); } @@ -2279,7 +2277,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) { static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - char* dstFilenameOutDir = NULL; + char* outDirFilename = NULL; size_t sfnSize = strlen(srcFileName); size_t suffixSize; @@ -2321,9 +2319,9 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) return NULL; } if (outDirName) { - dstFilenameOutDir = FIO_determineDstFilenameOutdir(srcFileName, outDirName, 0); - sfnSize = strlen(dstFilenameOutDir); - assert(dstFilenameOutDir != NULL); + outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0); + sfnSize = strlen(outDirFilename); + assert(outDirFilename != NULL); } if (dfnbCapacity+suffixSize <= sfnSize+1) { @@ -2337,9 +2335,9 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) /* return dst name == src name truncated from suffix */ assert(dstFileNameBuffer != NULL); - if (dstFilenameOutDir) { - memcpy(dstFileNameBuffer, dstFilenameOutDir, sfnSize - suffixSize); - free(dstFilenameOutDir); + if (outDirFilename) { + memcpy(dstFileNameBuffer, outDirFilename, sfnSize - suffixSize); + free(outDirFilename); } else { memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize); } From e0d413d648790fa3b34432a6bc2678a3e3361d58 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 4 Oct 2019 15:09:52 -0700 Subject: [PATCH 042/106] fixed init warning --- programs/util.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/util.c b/programs/util.c index 3988295d4..c5d344ec8 100644 --- a/programs/util.c +++ b/programs/util.c @@ -69,7 +69,8 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf) #else { /* (atime, mtime) */ - struct timespec timebuf[2] = { {0, UTIME_NOW}, statbuf->st_mtim }; + struct timespec timebuf[2] = { {0, UTIME_NOW} }; + timebuf[1] = statbuf->st_mtim; res += utimensat(AT_FDCWD, filename, timebuf, 0); } #endif From f9b1da6f62b265f38603e916fd6621198b862a20 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 8 Oct 2019 00:15:28 -0400 Subject: [PATCH 043/106] Hopefully fix VS compiler warning --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 6f1a2d678..cc495b858 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -658,7 +658,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { } } - qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + qsort((const char**)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); prevElem = filenameTableSorted[0]; for (u = 1; u < nbFiles; ++u) { if (strcmp(prevElem, filenameTableSorted[u]) == 0) { @@ -667,7 +667,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { prevElem = filenameTableSorted[u]; } - free(filenameTableSorted); + free((const char**)filenameTableSorted); return 0; } From 332b5988d8f080aca68901e371b70451f7061fd8 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 8 Oct 2019 09:44:24 -0400 Subject: [PATCH 044/106] Suppress C4090 warning --- programs/fileio.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index cc495b858..9ac65abcb 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -658,7 +658,13 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { } } - qsort((const char**)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + /* Silence VS Compiler bug with warning C4090 */ + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) + #pragma warning (push) + #pragma warning (disable : 4090) + #endif + + qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); prevElem = filenameTableSorted[0]; for (u = 1; u < nbFiles; ++u) { if (strcmp(prevElem, filenameTableSorted[u]) == 0) { @@ -667,7 +673,11 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { prevElem = filenameTableSorted[u]; } - free((const char**)filenameTableSorted); + free(filenameTableSorted); + + #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) + #pragma warning (pop) + #endif return 0; } From 6e406b55eeeaa3ce1175459fda4044cbf248e0d1 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Tue, 8 Oct 2019 09:54:59 -0400 Subject: [PATCH 045/106] Casting to void* to avoid C4090 warnings --- programs/fileio.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9ac65abcb..81633947f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -658,13 +658,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { } } - /* Silence VS Compiler bug with warning C4090 */ - #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) - #pragma warning (push) - #pragma warning (disable : 4090) - #endif - - qsort(filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); prevElem = filenameTableSorted[0]; for (u = 1; u < nbFiles; ++u) { if (strcmp(prevElem, filenameTableSorted[u]) == 0) { @@ -673,11 +667,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { prevElem = filenameTableSorted[u]; } - free(filenameTableSorted); - - #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) - #pragma warning (pop) - #endif + free((void*)filenameTableSorted); return 0; } From 8826f3b433f65294414697df2cef23511211f39e Mon Sep 17 00:00:00 2001 From: Hasnain Lakhani Date: Tue, 8 Oct 2019 14:13:45 -0700 Subject: [PATCH 046/106] Add OSS-Fuzz fuzzing badge This is basically redoing https://github.com/facebook/zstd/pull/1818 by @inferno-chromium from the OSS-Fuzz team --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 290341cc7..9c5f92013 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww [![Build status][AppveyorDevBadge]][AppveyorLink] [![Build status][CircleDevBadge]][CircleLink] [![Build status][CirrusDevBadge]][CirrusLink] +[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink] [travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite" [travisLink]: https://travis-ci.org/facebook/zstd @@ -24,6 +25,8 @@ a list of known ports and bindings is provided on [Zstandard homepage](http://ww [CircleLink]: https://circleci.com/gh/facebook/zstd [CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev [CirrusLink]: https://cirrus-ci.com/github/facebook/zstd +[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg +[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd ## Benchmarks From c5ebb370510b5a2c7fbeac332a2480f51e0c9070 Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Wed, 9 Oct 2019 09:39:52 -0400 Subject: [PATCH 047/106] Only check for filename collisions when using --output-dir-flat --- programs/fileio.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 81633947f..39cc70b9c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1553,7 +1553,8 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileN const char* const dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); } - FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles); + if (outDirName) + FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles); } FIO_freeCResources(ress); @@ -2375,7 +2376,8 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); } - FIO_checkFilenameCollisions(srcNamesTable ,nbFiles); + if (outDirName) + FIO_checkFilenameCollisions(srcNamesTable ,nbFiles); } FIO_freeDResources(ress); From 46ee10dfb5944e4a277e402da6ec2e1168ece697 Mon Sep 17 00:00:00 2001 From: Eric van Gyzen Date: Thu, 10 Oct 2019 09:21:29 -0500 Subject: [PATCH 048/106] Include errno in all fwrite error messages in fileio.c --- programs/fileio.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 8a45563d4..09687c67a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -769,7 +769,7 @@ FIO_compressGzFrame(cRess_t* ress, { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) - EXM_THROW(73, "Write error : cannot write to output file"); + EXM_THROW(73, "Write error : cannot write to output file : %s", strerror(errno)); outFileSize += decompBytes; strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; @@ -1523,7 +1523,7 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi if (!prefs->sparseFileSupport) { /* normal write */ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); if (sizeCheck != bufferSize) - EXM_THROW(70, "Write error : %s (cannot write decoded block)", + EXM_THROW(70, "Write error : cannot write decoded block : %s", strerror(errno)); return 0; } @@ -1554,7 +1554,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi ptrT += nb0T; { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file); if (sizeCheck != seg0SizeT) - EXM_THROW(73, "Write error : cannot write decoded block"); + EXM_THROW(73, "Write error : cannot write decoded block : %s", + strerror(errno)); } } ptrT += seg0SizeT; } @@ -1575,7 +1576,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi storedSkips = 0; { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file); if (sizeCheck != (size_t)(restEnd - restPtr)) - EXM_THROW(75, "Write error : cannot write decoded end of block"); + EXM_THROW(75, "Write error : cannot write decoded end of block : %s", + strerror(errno)); } } } } return storedSkips; @@ -1593,7 +1595,7 @@ FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) * so that skipped ones get implicitly translated as zero by FS */ { const char lastZeroByte[1] = { 0 }; if (fwrite(lastZeroByte, 1, 1, file) != 1) - EXM_THROW(69, "Write error : cannot write last zero"); + EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); } } } @@ -1612,7 +1614,7 @@ static int FIO_passThrough(FIO_prefs_t* const prefs, /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */ { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput); if (sizeCheck != alreadyLoaded) { - DISPLAYLEVEL(1, "Pass-through write error \n"); + DISPLAYLEVEL(1, "Pass-through write error : %s\n", strerror(errno)); return 1; } } @@ -1783,7 +1785,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { - DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); + DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno)); decodingError = 1; break; } outFileSize += decompBytes; @@ -1858,7 +1860,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { - DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); + DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno)); decodingError = 1; break; } outFileSize += decompBytes; @@ -1929,7 +1931,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, /* Write Block */ if (decodedBytes) { if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) { - DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); + DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno)); decodingError = 1; nextToLoad = 0; break; } filesize += decodedBytes; From dc1fb684bfbd5bd1400226cb2c9ee69084782a90 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 18:24:44 -0400 Subject: [PATCH 049/106] Remove Unused MEM_SKIP_MSAN Macro --- lib/common/mem.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 2b115ddb6..acc7231e9 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -74,12 +74,6 @@ void __msan_poison(const volatile void *a, size_t size); intptr_t __msan_test_shadow(const volatile void *x, size_t size); #endif -#if defined (MEMORY_SANITIZER) -# define MEM_SKIP_MSAN __attribute__((no_sanitize("memory"))) -#else -# define MEM_SKIP_MSAN -#endif - /*-************************************************************** * Basic Types From edb6d884a593bb80c66825c8a476e47ded2d617a Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 18:32:22 -0400 Subject: [PATCH 050/106] Detect Whether We're Being Compiled with ASAN --- lib/common/mem.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/common/mem.h b/lib/common/mem.h index acc7231e9..983b55edc 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -74,6 +74,19 @@ void __msan_poison(const volatile void *a, size_t size); intptr_t __msan_test_shadow(const volatile void *x, size_t size); #endif +/* detects whether we are being compiled under asan */ +#if defined (__has_feature) +# if __has_feature(address_sanitizer) +# define ADDRESS_SANITIZER 1 +# endif +#elif defined(__SANITIZE_ADDRESS__) +# define ADDRESS_SANITIZER 1 +#endif + +#if defined (ADDRESS_SANITIZER) +# include +#endif + /*-************************************************************** * Basic Types From 35c30d6ca7219a602dc4c54d33835e4e1be6fee5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 18:33:00 -0400 Subject: [PATCH 051/106] Poison Unused Workspace Memory --- lib/compress/zstd_cwksp.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 39d064c49..6d775f90b 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -214,6 +214,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( ws->tableValidEnd = alloc; } ws->allocStart = alloc; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + __asan_unpoison_memory_region(alloc, bytes); +#endif + return alloc; } @@ -254,6 +259,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { return NULL; } ws->tableEnd = end; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + __asan_unpoison_memory_region(alloc, bytes); +#endif + return alloc; } @@ -279,6 +289,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { ws->objectEnd = end; ws->tableEnd = end; ws->tableValidEnd = end; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + __asan_unpoison_memory_region(start, bytes); +#endif + return start; } @@ -331,6 +346,14 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { */ MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { DEBUGLOG(4, "cwksp: clearing tables!"); + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + ws->tableEnd = ws->objectEnd; ZSTD_cwksp_assert_internal_consistency(ws); } @@ -353,6 +376,13 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { } #endif +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + ws->tableEnd = ws->objectEnd; ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; From da88c35d41f53255f644a8b2ca27f4fa017048e6 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 18:39:46 -0400 Subject: [PATCH 052/106] Stop Assuming Tables are Adjacent --- lib/compress/zstd_compress.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2f0736b24..02011c946 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1649,12 +1649,13 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, /* copy tables */ { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); size_t const hSize = (size_t)1 << cdict_cParams->hashLog; - size_t const tableSpace = (chainSize + hSize) * sizeof(U32); - assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize); - assert((U32*)cdict->matchState.chainTable == (U32*)cdict->matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize); - memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, tableSpace); /* presumes all tables follow each other */ + + memcpy(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize * sizeof(U32)); + memcpy(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize * sizeof(U32)); } /* Zero the hashTable3, since the cdict never fills it */ @@ -1741,10 +1742,16 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; int const h3log = srcCCtx->blockState.matchState.hashLog3; size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - assert((U32*)dstCCtx->blockState.matchState.chainTable == (U32*)dstCCtx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)dstCCtx->blockState.matchState.hashTable3 == (U32*)dstCCtx->blockState.matchState.chainTable + chainSize); - memcpy(dstCCtx->blockState.matchState.hashTable, srcCCtx->blockState.matchState.hashTable, tableSpace); /* presumes all tables follow each other */ + + memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); } ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); From 19a0955ec90b30d249e253b47160d40f5b34a5ea Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 19:40:38 -0400 Subject: [PATCH 053/106] Add `ZSTD_cwksp_alloc_size()` to Help Calculate Needed Workspace Size --- lib/compress/zstd_compress.c | 58 ++++++++++++++++++++++-------------- lib/compress/zstd_cwksp.h | 25 ++++++++++++++++ lib/compress/zstd_ldm.c | 6 ++-- 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 02011c946..8fbddf8b2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1078,9 +1078,16 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<strategy >= ZSTD_btopt)) ? optPotentialSpace : 0; @@ -1097,20 +1104,21 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; - size_t const entropySpace = HUF_WORKSPACE_SIZE; - size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq); + size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams); - size_t const ldmSeqSpace = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq); + size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq)); size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace + matchStateSize + ldmSpace + ldmSeqSpace; + size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)); - DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); + DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)cctxSpace); DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); - return sizeof(ZSTD_CCtx) + neededSpace; + return cctxSpace + neededSpace; } } @@ -1393,7 +1401,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq); size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); @@ -1408,12 +1416,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_cwksp_bump_oversized_duration(ws, 0); /* Check if workspace is large enough, alloc a new one if needed */ - { size_t const cctxSpace = zc->staticSize ? sizeof(ZSTD_CCtx) : 0; - size_t const entropySpace = HUF_WORKSPACE_SIZE; - size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); - size_t const bufferSpace = buffInSize + buffOutSize; + { size_t const cctxSpace = zc->staticSize ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + ZSTD_cwksp_alloc_size(buffOutSize); size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); - size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); + size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)); size_t const neededSpace = cctxSpace + @@ -3060,8 +3068,11 @@ size_t ZSTD_estimateCDictSize_advanced( ZSTD_dictLoadMethod_e dictLoadMethod) { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); - return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void *))); + return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); } size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) @@ -3141,11 +3152,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, if (!customMem.customAlloc ^ !customMem.customFree) return NULL; { size_t const workspaceSize = - sizeof(ZSTD_CDict) + - HUF_WORKSPACE_SIZE + + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 - : ZSTD_cwksp_align(dictSize, sizeof(void*))); + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); ZSTD_cwksp ws; ZSTD_CDict* cdict; @@ -3224,8 +3235,11 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_compressionParameters cParams) { size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); - size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void*))) - + HUF_WORKSPACE_SIZE + matchStateSize; + size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + matchStateSize; ZSTD_CDict* cdict; if ((size_t)workspace & 7) return NULL; /* 8-aligned */ diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 6d775f90b..5dce662d7 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -34,6 +34,17 @@ extern "C" { * In which case, resize it down to free some memory */ #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 8 +#endif + /*-************************************* * Structures ***************************************/ @@ -166,6 +177,20 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { return (size + mask) & ~mask; } +/** + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + */ +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#else + return size; +#endif +} + MEM_STATIC void ZSTD_cwksp_internal_advance_phase( ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { assert(phase >= ws->phase); diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index fc3f46943..c3312ad3e 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -49,9 +49,9 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params) { size_t const ldmHSize = ((size_t)1) << params.hashLog; size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); - size_t const ldmBucketSize = - ((size_t)1) << (params.hashLog - ldmBucketSizeLog); - size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); return params.enableLdm ? totalSize : 0; } From 143b296cf6833c57934981e9c28d090a52caea17 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 19:41:45 -0400 Subject: [PATCH 054/106] Surround Workspace Allocs with Dead Zone --- lib/compress/zstd_cwksp.h | 49 ++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 5dce662d7..e35d8c1cd 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -226,8 +226,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( void* bottom = ws->tableEnd; ZSTD_cwksp_internal_advance_phase(ws, phase); alloc = (BYTE *)ws->allocStart - bytes; - DEBUGLOG(5, "cwksp: reserving %zd bytes, %zd bytes remaining", - bytes, ZSTD_cwksp_available_space(ws) - bytes); + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); ZSTD_cwksp_assert_internal_consistency(ws); assert(alloc >= bottom); if (alloc < bottom) { @@ -241,6 +247,9 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( ws->allocStart = alloc; #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; __asan_unpoison_memory_region(alloc, bytes); #endif @@ -272,8 +281,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { void* alloc = ws->tableEnd; void* end = (BYTE *)alloc + bytes; void* top = ws->allocStart; - DEBUGLOG(5, "cwksp: reserving table %zd bytes, %zd bytes remaining", - bytes, ZSTD_cwksp_available_space(ws) - bytes); + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); assert((bytes & (sizeof(U32)-1)) == 0); ZSTD_cwksp_internal_advance_phase(ws, phase); ZSTD_cwksp_assert_internal_consistency(ws); @@ -286,6 +301,9 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { ws->tableEnd = end; #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; __asan_unpoison_memory_region(alloc, bytes); #endif @@ -297,12 +315,18 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { */ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); - void* start = ws->objectEnd; - void* end = (BYTE*)start + roundedBytes; + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + DEBUGLOG(5, - "cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", - bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); - assert(((size_t)start & (sizeof(void*)-1)) == 0); + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert(((size_t)alloc & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); ZSTD_cwksp_assert_internal_consistency(ws); /* we must be in the first phase, no advance is possible */ @@ -316,10 +340,13 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { ws->tableValidEnd = end; #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - __asan_unpoison_memory_region(start, bytes); + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + __asan_unpoison_memory_region(alloc, bytes); #endif - return start; + return alloc; } MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { From ef0b5707c525f2bc97ba3c25119aaafc4d3896ab Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:43:05 -0400 Subject: [PATCH 055/106] Refactor Freeing CCtxes / CDicts Inside Workspaces --- lib/compress/zstd_compress.c | 20 +++++++++----------- lib/compress/zstd_cwksp.h | 7 +++++++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8fbddf8b2..431307f03 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -130,24 +130,23 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - /* Only free workspace if cctx not in workspace, otherwise the workspace - * will be freed when the cctx itself is freed. */ - if ((void*)cctx->workspace.workspace != (void*)cctx) { - ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); - } ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; #endif + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { + int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); if (cctx==NULL) return 0; /* support free on NULL */ RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "not compatible with static CCtx"); ZSTD_freeCCtxContent(cctx); - ZSTD_free(cctx, cctx->customMem); + if (!cctxInWorkspace) { + ZSTD_free(cctx, cctx->customMem); + } return 0; } @@ -3204,12 +3203,11 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; - /* Only free workspace if cdict not in workspace, otherwise the - * workspace will be freed when the cdict itself is freed. */ - if ((void*)cdict->workspace.workspace != (void*)cdict) { - ZSTD_cwksp_free(&cdict->workspace, cMem); + int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); + ZSTD_cwksp_free(&cdict->workspace, cMem); + if (!cdictInWorkspace) { + ZSTD_free(cdict, cMem); } - ZSTD_free(cdict, cMem); return 0; } } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index e35d8c1cd..f6068127b 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -217,6 +217,13 @@ MEM_STATIC void ZSTD_cwksp_internal_advance_phase( } } +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) { + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd); +} + /** * Internal function. Do not use directly. */ From 8cffd6ed082b996040ae8c64302e594b1368d272 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:43:55 -0400 Subject: [PATCH 056/106] Avoid ASAN Failure in ZSTD_cwksp_free() --- lib/compress/zstd_cwksp.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index f6068127b..bae5b654d 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -478,9 +478,10 @@ MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem } MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; DEBUGLOG(4, "cwksp: freeing workspace"); - ZSTD_free(ws->workspace, customMem); memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_free(ptr, customMem); } /** From b6c0a02a17bcab8aab56f58563f244c52cb1cab8 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:45:40 -0400 Subject: [PATCH 057/106] Fix ZSTD_sizeof_matchState() Calculation --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 431307f03..becdc728f 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1076,7 +1076,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = ((size_t)1) << hashLog3; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; size_t const tableSpace = ZSTD_cwksp_alloc_size(chainSize * sizeof(U32)) + ZSTD_cwksp_alloc_size(hSize * sizeof(U32)) + ZSTD_cwksp_alloc_size(h3Size * sizeof(U32)); From 0cc481ef66b951a681ba9af70e06c13b91075ada Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:47:29 -0400 Subject: [PATCH 058/106] Fix Workspace Size Calculation --- lib/compress/zstd_compress.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index becdc728f..5c7392908 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1400,7 +1400,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); From a07037b7846f473bb27018a860b9ad1c1c9d46b5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:56:28 -0400 Subject: [PATCH 059/106] Don't Try to Redzone the Tables --- lib/compress/zstd_cwksp.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index bae5b654d..2fca8cff2 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -289,11 +289,6 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { void* end = (BYTE *)alloc + bytes; void* top = ws->allocStart; -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* over-reserve space */ - end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; -#endif - DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); assert((bytes & (sizeof(U32)-1)) == 0); @@ -308,9 +303,6 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { ws->tableEnd = end; #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on - * either size. */ - alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; __asan_unpoison_memory_region(alloc, bytes); #endif From 0ffae7e44061bd50aa76cff8b310d6da2f802ddc Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 18:06:16 -0400 Subject: [PATCH 060/106] Stop Allocating Extra Space for Table Redzones --- lib/compress/zstd_compress.c | 6 +++--- lib/compress/zstd_cwksp.h | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 5c7392908..110066731 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1077,9 +1077,9 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; - size_t const tableSpace = ZSTD_cwksp_alloc_size(chainSize * sizeof(U32)) - + ZSTD_cwksp_alloc_size(hSize * sizeof(U32)) - + ZSTD_cwksp_alloc_size(h3Size * sizeof(U32)); + size_t const tableSpace = chainSize * sizeof(U32) + + hSize * sizeof(U32) + + h3Size * sizeof(U32); size_t const optPotentialSpace = ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32)) + ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32)) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 2fca8cff2..15b48b470 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -182,6 +182,10 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { * allocate this object. (Normally it should be exactly the size of the object, * but under special conditions, like ASAN, where we pad each object, it might * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. */ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) From b6987acbbf1d565feac9473eb095eebade7f6f1d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 19:04:05 -0400 Subject: [PATCH 061/106] Declare the ASAN Functions We Need, Don't Include the Header --- lib/common/mem.h | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 983b55edc..530d30c8f 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -84,7 +84,40 @@ intptr_t __msan_test_shadow(const volatile void *x, size_t size); #endif #if defined (ADDRESS_SANITIZER) -# include +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ + +/** + * Marks a memory region ([addr, addr+size)) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of [addr, addr+size) due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region ([addr, addr+size)) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of [addr, addr+size) due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); #endif From 2c80a9f8ac73ad6634dfe68ae1624eb1af5e3d61 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 17 Sep 2019 11:35:49 -0400 Subject: [PATCH 062/106] Check if CCtx in Workspace after Null Check --- lib/compress/zstd_compress.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 110066731..4afb9cdca 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -139,13 +139,15 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { - int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); if (cctx==NULL) return 0; /* support free on NULL */ RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "not compatible with static CCtx"); - ZSTD_freeCCtxContent(cctx); - if (!cctxInWorkspace) { - ZSTD_free(cctx, cctx->customMem); + { + int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) { + ZSTD_free(cctx, cctx->customMem); + } } return 0; } @@ -1077,6 +1079,8 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't + * surrounded by redzones in ASAN. */ size_t const tableSpace = chainSize * sizeof(U32) + hSize * sizeof(U32) + h3Size * sizeof(U32); From bd6a20b8a0cc16fbd6efa974bf60d59bb79f6398 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 10 Oct 2019 13:45:55 -0400 Subject: [PATCH 063/106] Expand Default Redzone Size --- lib/compress/zstd_cwksp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 15b48b470..18492acb4 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -42,7 +42,7 @@ extern "C" { * This defines the size of that redzone. */ #ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE -#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 8 +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 #endif /*-************************************* From ede31da2ea317d72eb0525efe865e7b2c3ce44b5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 10 Oct 2019 15:02:08 -0400 Subject: [PATCH 064/106] Fix CCtx Size Estimation --- lib/compress/zstd_compress.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4afb9cdca..7af657d74 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1107,7 +1107,9 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); @@ -1157,7 +1159,8 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; - size_t const streamingSize = inBuffSize + outBuffSize; + size_t const streamingSize = ZSTD_cwksp_alloc_size(inBuffSize) + + ZSTD_cwksp_alloc_size(outBuffSize); return CCtxSize + streamingSize; } From 6309be677c21c7fef359569a3e747a3934ab35c3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 15 Oct 2019 16:09:18 -0700 Subject: [PATCH 065/106] minor comments & refactoring --- programs/benchzstd.c | 39 +++++++++++++++++++-------------------- programs/benchzstd.h | 1 - 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 263dc0888..7439677c7 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -88,7 +88,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } -#define EXM_THROW_INT(errorNum, ...) { \ +#define RETURN_ERROR_INT(errorNum, ...) { \ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ DISPLAYLEVEL(1, "Error %i : ", errorNum); \ DISPLAYLEVEL(1, __VA_ARGS__); \ @@ -401,9 +401,9 @@ BMK_benchMemAdvancedNoAlloc( BMK_initCCtxArgs cctxprep; BMK_initDCtxArgs dctxprep; - cbp.benchFn = local_defaultCompress; + cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ cbp.benchPayload = cctx; - cbp.initFn = local_initCCtx; + cbp.initFn = local_initCCtx; /* BMK_initCCtx */ cbp.initPayload = &cctxprep; cbp.errorFn = ZSTD_isError; cbp.blockCount = nbBlocks; @@ -534,8 +534,8 @@ BMK_benchMemAdvancedNoAlloc( if (u==srcSize-1) { /* should never happen */ DISPLAY("no difference detected\n"); } - } - } + } /* for (u=0; umode == BMK_both) && (crcOrig!=crcCheck)) */ } /* CRC Checking */ if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ @@ -754,8 +754,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, size_t pos = 0, totalSize = 0; unsigned n; for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]); - pos += readSize; - } - fileSizes[n] = (size_t)fileSize; - totalSize += (size_t)fileSize; - fclose(f); - } + { FILE* const f = fopen(fileNamesTable[n], "rb"); + if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]); + DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); + if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]); + pos += readSize; + } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } } - if (totalSize == 0) EXM_THROW_INT(12, "no data to bench"); + if (totalSize == 0) RETURN_ERROR_INT(12, "no data to bench"); return 0; } diff --git a/programs/benchzstd.h b/programs/benchzstd.h index 2c7627713..ef7d9fb11 100644 --- a/programs/benchzstd.h +++ b/programs/benchzstd.h @@ -205,7 +205,6 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, - #endif /* BENCH_ZSTD_H_3242387 */ #if defined (__cplusplus) From 2d5201b0ab16b74e6aa1b8fa3079d3a5eb4f7eaf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 Oct 2019 14:51:33 -0700 Subject: [PATCH 066/106] removed wildcopy8() which is no longer used, noticed by @davidbolvansky --- doc/zstd_manual.html | 27 +++++++++++++++++++++++++++ lib/common/zstd_internal.h | 14 -------------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 79b9d0231..7a1b457d3 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -866,6 +866,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
typedef struct {
+    unsigned int matchPos; /* Match pos in dst */
+    /* If seqDef.offset > 3, then this is seqDef.offset - 3
+     * If seqDef.offset < 3, then this is the corresponding repeat offset
+     * But if seqDef.offset < 3 and litLength == 0, this is the
+     *   repeat offset before the corresponding repeat offset
+     * And if seqDef.offset == 3 and litLength == 0, this is the
+     *   most recent repeat offset - 1
+     */
+    unsigned int offset;
+    unsigned int litLength; /* Literal length */
+    unsigned int matchLength; /* Match length */
+    /* 0 when seq not rep and seqDef.offset otherwise
+     * when litLength == 0 this will be <= 4, otherwise <= 3 like normal
+     */
+    unsigned int rep;
+} ZSTD_Sequence;
+

typedef struct {
     unsigned windowLog;       /**< largest match distance : larger == more compression, more memory needed during decompression */
     unsigned chainLog;        /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
@@ -1001,6 +1019,15 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
            or an error code (if srcSize is too small)

+
size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
+    size_t outSeqsSize, const void* src, size_t srcSize);
+
 Extract sequences from the sequence store
+ zc can be used to insert custom compression params.
+ This function invokes ZSTD_compress2
+ @return : number of sequences extracted
+ 
+

+

Memory management


 
 
size_t ZSTD_estimateCCtxSize(int compressionLevel);
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index f791c5b38..13420bd8a 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -247,20 +247,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
     }
 }
 
-/*! ZSTD_wildcopy8() :
- *  The same as ZSTD_wildcopy(), but it can only overwrite 8 bytes, and works for
- *  overlapping buffers that are at least 8 bytes apart.
- */
-MEM_STATIC void ZSTD_wildcopy8(void* dst, const void* src, ptrdiff_t length)
-{
-    const BYTE* ip = (const BYTE*)src;
-    BYTE* op = (BYTE*)dst;
-    BYTE* const oend = (BYTE*)op + length;
-    do {
-        COPY8(op, ip);
-    } while (op < oend);
-}
-
 
 /*-*******************************************
 *  Private declarations

From 6323966e53955619e53f9d1d2993bbcae4c28629 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 16 Oct 2019 16:14:04 -0700
Subject: [PATCH 067/106] updated erroneous comments using ZSTD_dm_*

instead of the current ZSTD_dct_*,
reported by @nigeltao (#1822)
---
 doc/zstd_manual.html         | 35 +++++++++++++++++++++++++++++++----
 lib/compress/zstd_compress.c |  2 +-
 lib/zstd.h                   |  8 ++++----
 tests/fuzzer.c               |  8 ++++----
 4 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 79b9d0231..4a0d3ee37 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -796,7 +796,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
   Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
            It's a CPU consuming operation, with non-negligible impact on latency.
            If there is a need to use the same prefix multiple times, consider loadDictionary instead.
-  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
+  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
            Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. 
 


 
@@ -840,7 +840,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
   Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
            Prefix buffer must remain unmodified up to the end of frame,
            reached when ZSTD_decompressStream() returns 0.
-  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+  Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
            Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
   Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
            A full dictionary is more costly, as it requires building tables.
@@ -866,6 +866,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  
 

 
+
typedef struct {
+    unsigned int matchPos; /* Match pos in dst */
+    /* If seqDef.offset > 3, then this is seqDef.offset - 3
+     * If seqDef.offset < 3, then this is the corresponding repeat offset
+     * But if seqDef.offset < 3 and litLength == 0, this is the
+     *   repeat offset before the corresponding repeat offset
+     * And if seqDef.offset == 3 and litLength == 0, this is the
+     *   most recent repeat offset - 1
+     */
+    unsigned int offset;
+    unsigned int litLength; /* Literal length */
+    unsigned int matchLength; /* Match length */
+    /* 0 when seq not rep and seqDef.offset otherwise
+     * when litLength == 0 this will be <= 4, otherwise <= 3 like normal
+     */
+    unsigned int rep;
+} ZSTD_Sequence;
+


 
typedef struct {
     unsigned windowLog;       /**< largest match distance : larger == more compression, more memory needed during decompression */
     unsigned chainLog;        /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
@@ -1001,6 +1019,15 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
            or an error code (if srcSize is too small) 
 


 
+
size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
+    size_t outSeqsSize, const void* src, size_t srcSize);
+
 Extract sequences from the sequence store
+ zc can be used to insert custom compression params.
+ This function invokes ZSTD_compress2
+ @return : number of sequences extracted
+ 
+


+
 
Memory management

 
 
size_t ZSTD_estimateCCtxSize(int compressionLevel);
@@ -1322,7 +1349,7 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigne
  *
  * Creates of an internal CDict (incompatible with static CCtx), except if
  * dict == NULL or dictSize < 8, in which case no dict is used.
- * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
  * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
  */
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
@@ -1337,7 +1364,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
  *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
  *
  * pledgedSrcSize must be correct. If srcSize is not known at init time, use
- * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
+ * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
  */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 7facbeff0..11354e535 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3518,7 +3518,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 /* ZSTD_initCStream_advanced() :
  * pledgedSrcSize must be exact.
  * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
- * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
+ * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  const void* dict, size_t dictSize,
                                  ZSTD_parameters params, unsigned long long pss)
diff --git a/lib/zstd.h b/lib/zstd.h
index 667845627..a710a5105 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -928,7 +928,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
  *  Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
  *           It's a CPU consuming operation, with non-negligible impact on latency.
  *           If there is a need to use the same prefix multiple times, consider loadDictionary instead.
- *  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
+ *  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
  *           Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
                                  const void* prefix, size_t prefixSize);
@@ -972,7 +972,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
  *  Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
  *           Prefix buffer must remain unmodified up to the end of frame,
  *           reached when ZSTD_decompressStream() returns 0.
- *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+ *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
  *           Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
  *  Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
  *           A full dictionary is more costly, as it requires building tables.
@@ -1670,7 +1670,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
  *
  * Creates of an internal CDict (incompatible with static CCtx), except if
  * dict == NULL or dictSize < 8, in which case no dict is used.
- * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
  * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
  */
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
@@ -1685,7 +1685,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
  *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
  *
  * pledgedSrcSize must be correct. If srcSize is not known at init time, use
- * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
+ * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
  */
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 1f46363ad..a109a440d 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1387,7 +1387,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
-        DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
+        DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
             ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
             if (cdict==NULL) goto _output_error;
@@ -1395,7 +1395,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
-        DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
+        DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
             ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
             if (cdict!=NULL) goto _output_error;
@@ -1403,7 +1403,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
-        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++);
+        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
         {
             size_t ret;
             MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@@ -1417,7 +1417,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
-        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++);
+        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
         {
             size_t ret;
             MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);

From 83749411a65bda3c2075253ac740b3ffb5ea6c5f Mon Sep 17 00:00:00 2001
From: Bimba Shrestha 
Date: Wed, 16 Oct 2019 16:26:46 -0700
Subject: [PATCH 068/106] Removing unnecessary check from decode side

---
 doc/educational_decoder/zstd_decompress.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
index f3e1b848f..53ac52e42 100644
--- a/doc/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -856,8 +856,7 @@ static size_t decode_literals_compressed(frame_context_t *const ctx,
         // Impossible
         IMPOSSIBLE();
     }
-    if (regenerated_size > MAX_LITERALS_SIZE ||
-        compressed_size >= regenerated_size) {
+    if (regenerated_size > MAX_LITERALS_SIZE) {
         CORRUPTION();
     }
 

From 25ce9ac401ad153697ba34f6cd65c69dd9817add Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 16 Oct 2019 17:27:03 -0700
Subject: [PATCH 069/106] removed UNALIGNED() macro from educational decoder

as this name collides with existing macro in mingw64+clang9.
---
 doc/educational_decoder/zstd_decompress.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
index 53ac52e42..64e1b8738 100644
--- a/doc/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -1529,7 +1529,7 @@ void free_dictionary(dictionary_t *const dict) {
 /******* END DICTIONARY PARSING ***********************************************/
 
 /******* IO STREAM OPERATIONS *************************************************/
-#define UNALIGNED() ERROR("Attempting to operate on a non-byte aligned stream")
+
 /// Reads `num` bits from a bitstream, and updates the internal offset
 static inline u64 IO_read_bits(istream_t *const in, const int num_bits) {
     if (num_bits > 64 || num_bits <= 0) {
@@ -1608,7 +1608,7 @@ static inline const u8 *IO_get_read_ptr(istream_t *const in, size_t len) {
         INP_SIZE();
     }
     if (in->bit_offset != 0) {
-        UNALIGNED();
+        ERROR("Attempting to operate on a non-byte aligned stream");
     }
     const u8 *const ptr = in->ptr;
     in->ptr += len;
@@ -1634,7 +1634,7 @@ static inline void IO_advance_input(istream_t *const in, size_t len) {
          INP_SIZE();
     }
     if (in->bit_offset != 0) {
-        UNALIGNED();
+        ERROR("Attempting to operate on a non-byte aligned stream");
     }
 
     in->ptr += len;

From 303261f659959bcd665ce3ff3bc2e3d4c88ccc23 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 10:50:39 -0700
Subject: [PATCH 070/106] ignore build artifact from educational decoder test

---
 doc/educational_decoder/.gitignore | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 doc/educational_decoder/.gitignore

diff --git a/doc/educational_decoder/.gitignore b/doc/educational_decoder/.gitignore
new file mode 100644
index 000000000..b801306fa
--- /dev/null
+++ b/doc/educational_decoder/.gitignore
@@ -0,0 +1,2 @@
+# Build artifacts
+harness

From 000404311f60b39d45874bc7bd04672f54bad840 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 10:56:14 -0700
Subject: [PATCH 071/106] fix incorrect dictName/FileName comparison on Windows

inode identification does not seem to work on Windows,
even with on a msys2 posix layer.
---
 programs/util.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index 0908a43a2..735606fc0 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -106,20 +106,23 @@ int UTIL_compareStr(const void *p1, const void *p2) {
     return strcmp(* (char * const *) p1, * (char * const *) p2);
 }
 
-int UTIL_isSameFile(const char* file1, const char* file2)
+int UTIL_isSameFile(const char* fName1, const char* fName2)
 {
-#if defined(_MSC_VER)
+    assert(fName1 != NULL); assert(fName2 != NULL);
+#if defined(_MSC_VER) || defined(_WIN32)
     /* note : Visual does not support file identification by inode.
+     *        inode does not work on Windows, even with a posix layer, like msys2.
      *        The following work-around is limited to detecting exact name repetition only,
      *        aka `filename` is considered different from `subdir/../filename` */
     return !strcmp(file1, file2);
 #else
-    stat_t file1Stat;
-    stat_t file2Stat;
-    return UTIL_getFileStat(file1, &file1Stat)
-        && UTIL_getFileStat(file2, &file2Stat)
-        && (file1Stat.st_dev == file2Stat.st_dev)
-        && (file1Stat.st_ino == file2Stat.st_ino);
+    {   stat_t file1Stat;
+        stat_t file2Stat;
+        return UTIL_getFileStat(fName1, &file1Stat)
+            && UTIL_getFileStat(fName2, &file2Stat)
+            && (file1Stat.st_dev == file2Stat.st_dev)
+            && (file1Stat.st_ino == file2Stat.st_ino);
+    }
 #endif
 }
 

From a71256a2ee4fc219a85ca0c93260c30434b026ba Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 11:01:20 -0700
Subject: [PATCH 072/106] fix several cast

---
 programs/util.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index 735606fc0..321da066a 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -243,19 +243,20 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
     DIR *dir;
     struct dirent *entry;
     char* path;
-    int dirLength, fnameLength, pathLength, nbFiles = 0;
+    size_t dirLength, fnameLength, pathLength;
+    int nbFiles = 0;
 
     if (!(dir = opendir(dirName))) {
         UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
         return 0;
     }
 
-    dirLength = (int)strlen(dirName);
+    dirLength = strlen(dirName);
     errno = 0;
     while ((entry = readdir(dir)) != NULL) {
         if (strcmp (entry->d_name, "..") == 0 ||
             strcmp (entry->d_name, ".") == 0) continue;
-        fnameLength = (int)strlen(entry->d_name);
+        fnameLength = strlen(entry->d_name);
         path = (char*) malloc(dirLength + fnameLength + 2);
         if (!path) { closedir(dir); return 0; }
         memcpy(path, dirName, dirLength);
@@ -277,7 +278,8 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
         } else {
             if (*bufStart + *pos + pathLength >= *bufEnd) {
                 ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
-                *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+                assert(newListSize >= 0);
+                *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
                 *bufEnd = *bufStart + newListSize;
                 if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
             }
@@ -335,7 +337,8 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
             size_t const len = strlen(inputNames[i]);
             if (buf + pos + len >= bufend) {
                 ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
-                buf = (char*)UTIL_realloc(buf, newListSize);
+                assert(newListSize >= 0);
+                buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
                 bufend = buf + newListSize;
                 if (!buf) return NULL;
             }
@@ -345,7 +348,7 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
                 nbFiles++;
             }
         } else {
-            nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
+            nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
             if (buf == NULL) return NULL;
     }   }
 

From 1a18f1484fac179125912c58573d743814c83084 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 13:01:18 -0700
Subject: [PATCH 073/106] force compression during tests

to erase potentially remaining artifacts from previous runs
---
 doc/educational_decoder/Makefile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile
index b2ed9f33d..a7dc21307 100644
--- a/doc/educational_decoder/Makefile
+++ b/doc/educational_decoder/Makefile
@@ -29,17 +29,17 @@ harness: $(HARNESS_FILES)
 	$(CC) $(FLAGS) $^ -o $@
 
 clean:
-	@$(RM) -f harness
+	@$(RM) harness
 	@$(RM) -rf harness.dSYM
 
 test: harness
 	#
 	# Testing single-file decompression with educational decoder
 	#
-	@$(ZSTD) README.md -o tmp.zst
+	@$(ZSTD) -f README.md -o tmp.zst
 	@./harness tmp.zst tmp
 	@$(DIFF) -s tmp README.md
-	@$(RM) -f tmp*
+	@$(RM) tmp*
 	#
 	# Testing dictionary decompression with education decoder
 	#
@@ -47,8 +47,8 @@ test: harness
 	@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
                   harness.c zstd_decompress.c zstd_decompress.h README.md \
                   harness.c zstd_decompress.c zstd_decompress.h README.md
-	@$(ZSTD) -D dictionary README.md -o tmp.zst
+	@$(ZSTD) -f README.md -D dictionary -o tmp.zst
 	@./harness tmp.zst tmp dictionary
 	@$(DIFF) -s tmp README.md
-	@$(RM) -f tmp* dictionary
+	@$(RM) tmp* dictionary
 	@$(MAKE) clean

From bfd829f2544a1518a896ed474408575461453a5b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:03:20 -0700
Subject: [PATCH 074/106] minor: DIFF determination

use gdiff on SunOS
---
 doc/educational_decoder/Makefile  | 14 +++++++++---
 doc/educational_decoder/harness.c | 38 ++++++++++++++++---------------
 2 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile
index a7dc21307..704f86766 100644
--- a/doc/educational_decoder/Makefile
+++ b/doc/educational_decoder/Makefile
@@ -7,8 +7,15 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-ZSTD ?= zstd   # requires zstd installation on local system
+ZSTD ?= zstd   # note: requires zstd installation on local system
+
+UNAME?= $(shell uname)
+ifeq ($(UNAME), SunOS)
+DIFF ?= gdiff
+else
 DIFF ?= diff
+endif
+
 HARNESS_FILES=*.c
 
 MULTITHREAD_LDFLAGS = -pthread
@@ -30,7 +37,7 @@ harness: $(HARNESS_FILES)
 
 clean:
 	@$(RM) harness
-	@$(RM) -rf harness.dSYM
+	@$(RM) -rf harness.dSYM  # MacOS specific
 
 test: harness
 	#
@@ -46,7 +53,8 @@ test: harness
 	# note : files are presented multiple for training, to reach minimum threshold
 	@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
                   harness.c zstd_decompress.c zstd_decompress.h README.md \
-                  harness.c zstd_decompress.c zstd_decompress.h README.md
+                  harness.c zstd_decompress.c zstd_decompress.h README.md \
+                  -o dictionary
 	@$(ZSTD) -f README.md -D dictionary -o tmp.zst
 	@./harness tmp.zst tmp dictionary
 	@$(DIFF) -s tmp README.md
diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index 36f3967a9..df4c0af94 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -25,28 +25,29 @@ u8 *input;
 u8 *output;
 u8 *dict;
 
-size_t read_file(const char *path, u8 **ptr) {
-    FILE *f = fopen(path, "rb");
+static size_t read_file(const char *path, u8 **ptr)
+{
+    FILE* const f = fopen(path, "rb");
     if (!f) {
-        fprintf(stderr, "failed to open file %s\n", path);
+        fprintf(stderr, "failed to open file %s \n", path);
         exit(1);
     }
 
     fseek(f, 0L, SEEK_END);
-    size_t size = (size_t)ftell(f);
+    size_t const size = (size_t)ftell(f);
     rewind(f);
 
     *ptr = malloc(size);
     if (!ptr) {
-        fprintf(stderr, "failed to allocate memory to hold %s\n", path);
+        fprintf(stderr, "failed to allocate memory to hold %s \n", path);
         exit(1);
     }
 
     size_t pos = 0;
     while (!feof(f)) {
-        size_t read = fread(&(*ptr)[pos], 1, size, f);
+        size_t const read = fread(*ptr + pos, 1, size, f);
         if (ferror(f)) {
-            fprintf(stderr, "error while reading file %s\n", path);
+            fprintf(stderr, "error while reading file %s \n", path);
             exit(1);
         }
         pos += read;
@@ -57,30 +58,30 @@ size_t read_file(const char *path, u8 **ptr) {
     return pos;
 }
 
-void write_file(const char *path, const u8 *ptr, size_t size) {
-    FILE *f = fopen(path, "wb");
+static void write_file(const char *path, const u8 *ptr, size_t size)
+{
+    FILE* const f = fopen(path, "wb");
 
     size_t written = 0;
     while (written < size) {
-        written += fwrite(&ptr[written], 1, size, f);
+        written += fwrite(ptr+written, 1, size, f);
         if (ferror(f)) {
             fprintf(stderr, "error while writing file %s\n", path);
             exit(1);
-        }
-    }
+    }   }
 
     fclose(f);
 }
 
 int main(int argc, char **argv) {
     if (argc < 3) {
-        fprintf(stderr, "usage: %s   [dictionary]\n",
+        fprintf(stderr, "usage: %s   [dictionary] \n",
                 argv[0]);
 
         return 1;
     }
 
-    size_t input_size = read_file(argv[1], &input);
+    size_t const input_size = read_file(argv[1], &input);
     size_t dict_size = 0;
     if (argc >= 4) {
         dict_size = read_file(argv[3], &dict);
@@ -92,17 +93,17 @@ int main(int argc, char **argv) {
         fprintf(stderr, "WARNING: Compressed data does not contain "
                         "decompressed size, going to assume the compression "
                         "ratio is at most %d (decompressed size of at most "
-                        "%zu)\n",
-                MAX_COMPRESSION_RATIO, decompressed_size);
+                        "%u) \n",
+                MAX_COMPRESSION_RATIO, (unsigned)decompressed_size);
     }
     if (decompressed_size > MAX_OUTPUT_SIZE) {
         fprintf(stderr,
-                "Required output size too large for this implementation\n");
+                "Required output size too large for this implementation \n");
         return 1;
     }
     output = malloc(decompressed_size);
     if (!output) {
-        fprintf(stderr, "failed to allocate memory\n");
+        fprintf(stderr, "failed to allocate memory \n");
         return 1;
     }
 
@@ -122,4 +123,5 @@ int main(int argc, char **argv) {
     free(output);
     free(dict);
     input = output = dict = NULL;
+    return 0;
 }

From b062b6fb2d2c5e58c47fdac4355449051445fd6c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:11:54 -0700
Subject: [PATCH 075/106] minor refactoring of harness

---
 doc/educational_decoder/harness.c | 37 ++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index df4c0af94..338394916 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -21,10 +21,6 @@ typedef unsigned char u8;
 // Protect against allocating too much memory for output
 #define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024)
 
-u8 *input;
-u8 *output;
-u8 *dict;
-
 static size_t read_file(const char *path, u8 **ptr)
 {
     FILE* const f = fopen(path, "rb");
@@ -61,6 +57,10 @@ static size_t read_file(const char *path, u8 **ptr)
 static void write_file(const char *path, const u8 *ptr, size_t size)
 {
     FILE* const f = fopen(path, "wb");
+    if (!f) {
+        fprintf(stderr, "failed to open file %s \n", path);
+        exit(1);
+    }
 
     size_t written = 0;
     while (written < size) {
@@ -73,7 +73,8 @@ static void write_file(const char *path, const u8 *ptr, size_t size)
     fclose(f);
 }
 
-int main(int argc, char **argv) {
+int main(int argc, char **argv)
+{
     if (argc < 3) {
         fprintf(stderr, "usage: %s   [dictionary] \n",
                 argv[0]);
@@ -81,27 +82,31 @@ int main(int argc, char **argv) {
         return 1;
     }
 
+    u8* input;
     size_t const input_size = read_file(argv[1], &input);
+
+    u8* dict;
     size_t dict_size = 0;
     if (argc >= 4) {
         dict_size = read_file(argv[3], &dict);
     }
 
-    size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size);
-    if (decompressed_size == (size_t)-1) {
-        decompressed_size = MAX_COMPRESSION_RATIO * input_size;
+    size_t out_capacity = ZSTD_get_decompressed_size(input, input_size);
+    if (out_capacity == (size_t)-1) {
+        out_capacity = MAX_COMPRESSION_RATIO * input_size;
         fprintf(stderr, "WARNING: Compressed data does not contain "
                         "decompressed size, going to assume the compression "
                         "ratio is at most %d (decompressed size of at most "
                         "%u) \n",
-                MAX_COMPRESSION_RATIO, (unsigned)decompressed_size);
+                MAX_COMPRESSION_RATIO, (unsigned)out_capacity);
     }
-    if (decompressed_size > MAX_OUTPUT_SIZE) {
+    if (out_capacity > MAX_OUTPUT_SIZE) {
         fprintf(stderr,
                 "Required output size too large for this implementation \n");
         return 1;
     }
-    output = malloc(decompressed_size);
+
+    u8* const output = malloc(out_capacity);
     if (!output) {
         fprintf(stderr, "failed to allocate memory \n");
         return 1;
@@ -111,17 +116,17 @@ int main(int argc, char **argv) {
     if (dict) {
         parse_dictionary(parsed_dict, dict, dict_size);
     }
-    size_t decompressed =
-        ZSTD_decompress_with_dict(output, decompressed_size,
-                                  input, input_size, parsed_dict);
+    size_t const decompressed_size =
+        ZSTD_decompress_with_dict(output, out_capacity,
+                                  input, input_size,
+                                  parsed_dict);
 
     free_dictionary(parsed_dict);
 
-    write_file(argv[2], output, decompressed);
+    write_file(argv[2], output, decompressed_size);
 
     free(input);
     free(output);
     free(dict);
-    input = output = dict = NULL;
     return 0;
 }

From a0c041612d6ec63bf3ad028d928f7ee346956ed7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:15:00 -0700
Subject: [PATCH 076/106] fixed dict ptr init

---
 doc/educational_decoder/harness.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index 338394916..de2e12358 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -85,7 +85,7 @@ int main(int argc, char **argv)
     u8* input;
     size_t const input_size = read_file(argv[1], &input);
 
-    u8* dict;
+    u8* dict = NULL;
     size_t dict_size = 0;
     if (argc >= 4) {
         dict_size = read_file(argv[3], &dict);

From 5b8e873357a32e53edb6ef3b7cbe8bc58a9c75fc Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:29:48 -0700
Subject: [PATCH 077/106] fix harness test

---
 doc/educational_decoder/harness.c | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index de2e12358..a704f6bdb 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -39,19 +39,15 @@ static size_t read_file(const char *path, u8 **ptr)
         exit(1);
     }
 
-    size_t pos = 0;
-    while (!feof(f)) {
-        size_t const read = fread(*ptr + pos, 1, size, f);
-        if (ferror(f)) {
-            fprintf(stderr, "error while reading file %s \n", path);
-            exit(1);
-        }
-        pos += read;
+    size_t const read = fread(*ptr, 1, size, f);
+    if (read != size) {  /* must read everything in one pass */
+        fprintf(stderr, "error while reading file %s \n", path);
+        exit(1);
     }
 
     fclose(f);
 
-    return pos;
+    return read;
 }
 
 static void write_file(const char *path, const u8 *ptr, size_t size)

From 157479af0cc0611bedf3c5c1987971b761620eb1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:31:42 -0700
Subject: [PATCH 078/106] fixed isSameFile()

---
 programs/util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/util.c b/programs/util.c
index 321da066a..dbdd1cee4 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -114,7 +114,7 @@ int UTIL_isSameFile(const char* fName1, const char* fName2)
      *        inode does not work on Windows, even with a posix layer, like msys2.
      *        The following work-around is limited to detecting exact name repetition only,
      *        aka `filename` is considered different from `subdir/../filename` */
-    return !strcmp(file1, file2);
+    return !strcmp(fName1, fName2);
 #else
     {   stat_t file1Stat;
         stat_t file2Stat;

From ba7e2b6da73ca9dc9279357d8649e075c35030ce Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 15:07:47 -0700
Subject: [PATCH 079/106] tests: can override isTerminal with environment
 variable

---
 tests/playTests.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 06de4f8ac..f63a7f69e 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -64,11 +64,12 @@ PRGDIR="$SCRIPT_DIR/../programs"
 TESTDIR="$SCRIPT_DIR/../tests"
 UNAME=$(uname)
 
-isTerminal=false
+detectedTerminal=false
 if [ -t 0 ] && [ -t 1 ]
 then
-    isTerminal=true
+    detectedTerminal=true
 fi
+isTerminal=${isTerminal:-$detectedTerminal}
 
 isWindows=false
 INTOVOID="/dev/null"

From 7f86ae2867204c8f12fa8cfe858da2d62371370d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 15:27:25 -0700
Subject: [PATCH 080/106] fixed multiple implicit casts

---
 programs/zstdcli.c | 52 +++++++++++++++++++++++-----------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index ae53d2c39..6c6c10f28 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -287,7 +287,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
  *  If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
  * @return 0 and doesn't modify *stringPtr otherwise.
  */
-static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+static int longCommandWArg(const char** stringPtr, const char* longCommand)
 {
     size_t const comSize = strlen(longCommand);
     int const result = !strncmp(*stringPtr, longCommand, comSize);
@@ -430,8 +430,8 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void)
 static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
 {
     for ( ; ;) {
-        if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
-        if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         DISPLAYLEVEL(4, "invalid compression parameter \n");
         return 0;
     }
@@ -526,7 +526,7 @@ static int init_cLevel(void) {
                 DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
                 return ZSTDCLI_CLEVEL_DEFAULT;
             } else if (*ptr == 0) {
-                return sign * absLevel;
+                return sign * (int)absLevel;
             }
         }
 
@@ -584,7 +584,7 @@ int main(int argCount, const char* argv[])
     int cLevelLast = -1000000000;
     unsigned recursive = 0;
     unsigned memLimit = 0;
-    const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));   /* argCount >= 1 */
+    const char** filenameTable = (const char**)malloc((size_t)argCount * sizeof(const char*));   /* argCount >= 1 */
     unsigned filenameIdx = 0;
     const char* programName = argv[0];
     const char* outFileName = NULL;
@@ -745,7 +745,7 @@ int main(int argCount, const char* argv[])
                       continue;
                     }
 #endif
-                    if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; }
+                    if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
@@ -807,7 +807,7 @@ int main(int argCount, const char* argv[])
 #ifndef ZSTD_NOCOMPRESS
                     /* compression Level */
                     if ((*argument>='0') && (*argument<='9')) {
-                        dictCLevel = cLevel = readU32FromChar(&argument);
+                        dictCLevel = cLevel = (int)readU32FromChar(&argument);
                         continue;
                     }
 #endif
@@ -856,7 +856,7 @@ int main(int argCount, const char* argv[])
 
                         /* destination file name */
                     case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
-                   
+
                         /* limit decompression memory */
                     case 'M':
                         argument++;
@@ -879,7 +879,7 @@ int main(int argCount, const char* argv[])
                     case 'e':
                         /* compression Level */
                         argument++;
-                        cLevelLast = readU32FromChar(&argument);
+                        cLevelLast = (int)readU32FromChar(&argument);
                         break;
 
                         /* Modify Nb Iterations (benchmark only) */
@@ -905,7 +905,7 @@ int main(int argCount, const char* argv[])
                         /* nb of threads (hidden option) */
                     case 'T':
                         argument++;
-                        nbWorkers = readU32FromChar(&argument);
+                        nbWorkers = (int)readU32FromChar(&argument);
                         break;
 
                         /* Dictionary Selection level */
@@ -1042,16 +1042,16 @@ int main(int argCount, const char* argv[])
 #ifndef ZSTD_NOBENCH
         benchParams.blockSize = blockSize;
         benchParams.nbWorkers = nbWorkers;
-        benchParams.realTime = setRealTimePrio;
+        benchParams.realTime = (unsigned)setRealTimePrio;
         benchParams.nbSeconds = bench_nbSeconds;
         benchParams.ldmFlag = ldmFlag;
-        benchParams.ldmMinMatch = g_ldmMinMatch;
-        benchParams.ldmHashLog = g_ldmHashLog;
+        benchParams.ldmMinMatch = (int)g_ldmMinMatch;
+        benchParams.ldmHashLog = (int)g_ldmHashLog;
         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
-            benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
+            benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
         }
         if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
-            benchParams.ldmHashRateLog = g_ldmHashRateLog;
+            benchParams.ldmHashRateLog = (int)g_ldmHashRateLog;
         }
         benchParams.literalCompressionMode = literalCompressionMode;
 
@@ -1092,16 +1092,16 @@ int main(int argCount, const char* argv[])
 #ifndef ZSTD_NODICT
         ZDICT_params_t zParams;
         zParams.compressionLevel = dictCLevel;
-        zParams.notificationLevel = g_displayLevel;
+        zParams.notificationLevel = (unsigned)g_displayLevel;
         zParams.dictID = dictID;
         if (dict == cover) {
             int const optimize = !coverParams.k || !coverParams.d;
-            coverParams.nbThreads = nbWorkers;
+            coverParams.nbThreads = (unsigned)nbWorkers;
             coverParams.zParams = zParams;
             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
         } else if (dict == fastCover) {
             int const optimize = !fastCoverParams.k || !fastCoverParams.d;
-            fastCoverParams.nbThreads = nbWorkers;
+            fastCoverParams.nbThreads = (unsigned)nbWorkers;
             fastCoverParams.zParams = zParams;
             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
         } else {
@@ -1156,14 +1156,14 @@ int main(int argCount, const char* argv[])
     if (operation==zom_compress) {
 #ifndef ZSTD_NOCOMPRESS
         FIO_setNbWorkers(prefs, nbWorkers);
-        FIO_setBlockSize(prefs, (U32)blockSize);
-        if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog);
-        FIO_setLdmFlag(prefs, ldmFlag);
-        FIO_setLdmHashLog(prefs, g_ldmHashLog);
-        FIO_setLdmMinMatch(prefs, g_ldmMinMatch);
-        if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog);
-        if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog);
-        FIO_setAdaptiveMode(prefs, adapt);
+        FIO_setBlockSize(prefs, (int)blockSize);
+        if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
+        FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
+        FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
+        FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
+        if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
+        if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
+        FIO_setAdaptiveMode(prefs, (unsigned)adapt);
         FIO_setAdaptMin(prefs, adaptMin);
         FIO_setAdaptMax(prefs, adaptMax);
         FIO_setRsyncable(prefs, rsyncable);

From 1795133c450e839aa7b766d30a8c33989b7aad56 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 15:32:03 -0700
Subject: [PATCH 081/106] refactored FIO_compressMultipleFilenames() prototype

for consistency
---
 lib/compress/zstd_cwksp.h |  2 +-
 programs/fileio.c         | 17 +++++++++--------
 programs/fileio.h         |  9 +++++----
 programs/zstdcli.c        |  2 +-
 4 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index 39d064c49..a6a4a3fb8 100644
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -404,7 +404,7 @@ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
 }
 
 MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
-    return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace;
+    return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
 }
 
 MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
diff --git a/programs/fileio.c b/programs/fileio.c
index eecdf0dd3..f05275286 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -648,7 +648,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
         DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
         return 1;
     }
-    
+
     for (u = 0; u < nbFiles; ++u) {
         filename = strrchr(filenameTable[u], c[0]);
         if (filename == NULL) {
@@ -715,7 +715,7 @@ FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, c
     strcpy(result, outDirName);
     if (outDirName[strlen(outDirName)-1] == c[0]) {
         strcat(result, filename);
-    } else {  
+    } else {
         strcat(result, c);
         strcat(result, filename);
     }
@@ -1493,7 +1493,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
         sfnSize = strlen(outDirFilename);
         assert(outDirFilename != NULL);
     }
-    
+
     if (dfnbCapacity <= sfnSize+suffixSize+1) {
         /* resize buffer for dstName */
         free(dstFileNameBuffer);
@@ -1522,9 +1522,10 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
  * or into one file each (outFileName == NULL, but suffix != NULL),
  * or into a destination folder (specified with -O)
  */
-int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
-                                  const char* outDirName, unsigned nbFiles, 
-                                  const char* outFileName, const char* suffix, 
+int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
+                                  const char** inFileNamesTable, unsigned nbFiles,
+                                  const char* outDirName,
+                                  const char* outFileName, const char* suffix,
                                   const char* dictFileName, int compressionLevel,
                                   ZSTD_compressionParameters comprParams)
 {
@@ -2278,7 +2279,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
     char* outDirFilename = NULL;
     size_t sfnSize = strlen(srcFileName);
     size_t suffixSize;
-    
+
     const char* const suffixPtr = strrchr(srcFileName, '.');
     if (suffixPtr == NULL) {
         DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
@@ -2328,7 +2329,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
         dfnbCapacity = sfnSize + 20;
         dstFileNameBuffer = (char*)malloc(dfnbCapacity);
         if (dstFileNameBuffer==NULL)
-            EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno)); 
+            EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
     }
 
     /* return dst name == src name truncated from suffix */
diff --git a/programs/fileio.h b/programs/fileio.h
index ded1c0373..841e0a3c1 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -104,10 +104,11 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
 ***************************************/
 /** FIO_compressMultipleFilenames() :
     @return : nb of missing files */
-int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
-                                  const char* outDirName, unsigned nbFiles, 
-                                  const char* outFileName, const char* suffix, 
-                                  const char* dictFileName, int compressionLevel, 
+int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
+                                  const char** inFileNamesTable, unsigned nbFiles,
+                                  const char* outDirName,
+                                  const char* outFileName, const char* suffix,
+                                  const char* dictFileName, int compressionLevel,
                                   ZSTD_compressionParameters comprParams);
 
 /** FIO_decompressMultipleFilenames() :
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 6c6c10f28..4d1fad706 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1177,7 +1177,7 @@ int main(int argCount, const char* argv[])
         if ((filenameIdx==1) && outFileName)
           operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
         else
-          operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
+          operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
         (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
         DISPLAY("Compression not supported \n");

From 0ee360982d1b8e4f959f6da32bf1608935a867e8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 16:09:53 -0700
Subject: [PATCH 082/106] improved test mode `-t`

The test mode do no longer open a file `/dev/null` nor write anything to output.

This is supposed to be more efficient than writing to `/dev/null`,
and more universal too : the previous method was failing on Windows.
---
 programs/fileio.c  | 91 ++++++++++++++++++++++++++++------------------
 programs/fileio.h  | 13 ++++---
 programs/zstdcli.c |  2 +-
 3 files changed, 64 insertions(+), 42 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index f05275286..9efb4aaed 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -284,9 +284,10 @@ void FIO_addAbortHandler()
 
 
 /*-*************************************
-*  Parameters: Typedefs
+*  Parameters: FIO_prefs_t
 ***************************************/
 
+/* typedef'd to FIO_prefs_t within fileio.h */
 struct FIO_prefs_s {
 
     /* Algorithm preferences */
@@ -308,6 +309,7 @@ struct FIO_prefs_s {
     size_t streamSrcSize;
     size_t targetCBlockSize;
     int srcSizeHint;
+    int testMode;
     ZSTD_literalCompressionMode_e literalCompressionMode;
 
     /* IO preferences */
@@ -355,6 +357,7 @@ FIO_prefs_t* FIO_createPreferences(void)
     ret->streamSrcSize = 0;
     ret->targetCBlockSize = 0;
     ret->srcSizeHint = 0;
+    ret->testMode = 0;
     ret->literalCompressionMode = ZSTD_lcm_auto;
     return ret;
 }
@@ -435,6 +438,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) {
     prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint);
 }
 
+void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
+    prefs->testMode = (testMode!=0);
+}
+
 void FIO_setLiteralCompressionMode(
         FIO_prefs_t* const prefs,
         ZSTD_literalCompressionMode_e mode) {
@@ -1618,7 +1625,11 @@ static void FIO_freeDResources(dRess_t ress)
 
 /** FIO_fwriteSparse() :
 *   @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
-static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
+static unsigned
+FIO_fwriteSparse(const FIO_prefs_t* const prefs,
+                 FILE* file,
+                 const void* buffer, size_t bufferSize,
+                 unsigned storedSkips)
 {
     const size_t* const bufferT = (const size_t*)buffer;   /* Buffer is supposed malloc'ed, hence aligned on size_t */
     size_t bufferSizeT = bufferSize / sizeof(size_t);
@@ -1626,6 +1637,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
     const size_t* ptrT = bufferT;
     static const size_t segmentSizeT = (32 KB) / sizeof(size_t);   /* 0-test re-attempted every 32 KB */
 
+    if (prefs->testMode) return 0;  /* do not output anything in test mode */
+
     if (!prefs->sparseFileSupport) {  /* normal write */
         size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
         if (sizeCheck != bufferSize)
@@ -1690,8 +1703,9 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
 }
 
 static void
-FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
+FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
 {
+    if (prefs->testMode) assert(storedSkips == 0);
     if (storedSkips>0) {
         assert(prefs->sparseFileSupport > 0);  /* storedSkips>0 implies sparse support is enabled */
         (void)prefs;   /* assert can be disabled, in which case prefs becomes unused */
@@ -1708,7 +1722,7 @@ FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
 
 /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
     @return : 0 (no error) */
-static int FIO_passThrough(FIO_prefs_t* const prefs,
+static int FIO_passThrough(const FIO_prefs_t* const prefs,
                            FILE* foutput, FILE* finput,
                            void* buffer, size_t bufferSize,
                            size_t alreadyLoaded)
@@ -1748,7 +1762,10 @@ static unsigned FIO_highbit64(unsigned long long v)
 
 /* FIO_zstdErrorHelp() :
  * detailed error message when requested window size is too large */
-static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName)
+static void
+FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
+                  const dRess_t* ress,
+                  size_t err, const char* srcFileName)
 {
     ZSTD_frameHeader header;
 
@@ -1780,12 +1797,10 @@ static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t er
  *  @return : size of decoded zstd frame, or an error code
 */
 #define FIO_ERROR_FRAME_DECODING   ((unsigned long long)(-2))
-static unsigned long long FIO_decompressZstdFrame(
-                                       FIO_prefs_t* const prefs,
-                                       dRess_t* ress,
-                                       FILE* finput,
-                                       const char* srcFileName,
-                                       U64 alreadyDecoded)
+static unsigned long long
+FIO_decompressZstdFrame(const FIO_prefs_t* const prefs,
+                        dRess_t* ress, FILE* finput,
+                        const char* srcFileName, U64 alreadyDecoded)
 {
     U64 frameSize = 0;
     U32 storedSkips = 0;
@@ -1849,13 +1864,16 @@ static unsigned long long FIO_decompressZstdFrame(
 
 
 #ifdef ZSTD_GZDECOMPRESS
-static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
-                                    FILE* srcFile, const char* srcFileName)
+static unsigned long long
+FIO_decompressGzFrame(const FIO_prefs_t* const prefs,
+                      dRess_t* ress, FILE* srcFile,
+                      const char* srcFileName)
 {
     unsigned long long outFileSize = 0;
     z_stream strm;
     int flush = Z_NO_FLUSH;
     int decodingError = 0;
+    unsigned storedSkips = 0;
 
     strm.zalloc = Z_NULL;
     strm.zfree = Z_NULL;
@@ -1890,10 +1908,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
         }
         {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
             if (decompBytes) {
-                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
-                    DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
-                    decodingError = 1; break;
-                }
+                storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
                 outFileSize += decompBytes;
                 strm.next_out = (Bytef*)ress->dstBuffer;
                 strm.avail_out = (uInt)ress->dstBufferSize;
@@ -1910,19 +1925,24 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
         DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName);
         decodingError = 1;
     }
+    FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
     return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
 }
 #endif
 
 
 #ifdef ZSTD_LZMADECOMPRESS
-static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma)
+static unsigned long long
+FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs,
+                        dRess_t* ress, FILE* srcFile,
+                        const char* srcFileName, int plain_lzma)
 {
     unsigned long long outFileSize = 0;
     lzma_stream strm = LZMA_STREAM_INIT;
     lzma_action action = LZMA_RUN;
     lzma_ret initRet;
     int decodingError = 0;
+    unsigned storedSkips = 0;
 
     strm.next_in = 0;
     strm.avail_in = 0;
@@ -1965,10 +1985,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
         }
         {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
             if (decompBytes) {
-                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
-                    DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
-                    decodingError = 1; break;
-                }
+                storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
                 outFileSize += decompBytes;
                 strm.next_out = (BYTE*)ress->dstBuffer;
                 strm.avail_out = ress->dstBufferSize;
@@ -1980,19 +1997,23 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
         memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
     ress->srcBufferLoaded = strm.avail_in;
     lzma_end(&strm);
+    FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
     return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
 }
 #endif
 
 #ifdef ZSTD_LZ4DECOMPRESS
-static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
-                                    FILE* srcFile, const char* srcFileName)
+static unsigned long long
+FIO_decompressLz4Frame(const FIO_prefs_t* const prefs,
+                       dRess_t* ress, FILE* srcFile,
+                       const char* srcFileName)
 {
     unsigned long long filesize = 0;
     LZ4F_errorCode_t nextToLoad;
     LZ4F_decompressionContext_t dCtx;
     LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
     int decodingError = 0;
+    unsigned storedSkips = 0;
 
     if (LZ4F_isError(errorCode)) {
         DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n");
@@ -2036,10 +2057,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
 
             /* Write Block */
             if (decodedBytes) {
-                if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) {
-                    DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
-                    decodingError = 1; nextToLoad = 0; break;
-                }
+                storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decodedBytes, storedSkips);
                 filesize += decodedBytes;
                 DISPLAYUPDATE(2, "\rDecompressed : %u MB  ", (unsigned)(filesize>>20));
             }
@@ -2060,6 +2078,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
 
     LZ4F_freeDecompressionContext(dCtx);
     ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */
+    FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
 
     return decodingError ? FIO_ERROR_FRAME_DECODING : filesize;
 }
@@ -2073,8 +2092,9 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
  * @return : 0 : OK
  *           1 : error
  */
-static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile,
-                        const char* dstFileName, const char* srcFileName)
+static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
+                                dRess_t ress, FILE* srcFile,
+                          const char* dstFileName, const char* srcFileName)
 {
     unsigned readSomething = 0;
     unsigned long long filesize = 0;
@@ -2106,7 +2126,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
             filesize += frameSize;
         } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
 #ifdef ZSTD_GZDECOMPRESS
-            unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
+            unsigned long long const frameSize = FIO_decompressGzFrame(prefs, &ress, srcFile, srcFileName);
             if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
             filesize += frameSize;
 #else
@@ -2116,7 +2136,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
         } else if ((buf[0] == 0xFD && buf[1] == 0x37)  /* xz magic number */
                 || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
 #ifdef ZSTD_LZMADECOMPRESS
-            unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
+            unsigned long long const frameSize = FIO_decompressLzmaFrame(prefs, &ress, srcFile, srcFileName, buf[0] != 0xFD);
             if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
             filesize += frameSize;
 #else
@@ -2125,7 +2145,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
 #endif
         } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
 #ifdef ZSTD_LZ4DECOMPRESS
-            unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName);
+            unsigned long long const frameSize = FIO_decompressLz4Frame(prefs, &ress, srcFile, srcFileName);
             if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
             filesize += frameSize;
 #else
@@ -2165,11 +2185,11 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
     int transfer_permissions = 0;
     int releaseDstFile = 0;
 
-    if (ress.dstFile == NULL) {
+    if ((ress.dstFile == NULL) && (prefs->testMode==0)) {
         releaseDstFile = 1;
 
         ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
-        if (ress.dstFile==0) return 1;
+        if (ress.dstFile==NULL) return 1;
 
         /* Must only be added after FIO_openDstFile() succeeds.
          * Otherwise we may delete the destination file if it already exists,
@@ -2182,7 +2202,6 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
             transfer_permissions = 1;
     }
 
-
     result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName);
 
     if (releaseDstFile) {
diff --git a/programs/fileio.h b/programs/fileio.h
index 841e0a3c1..cb3d64c56 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -74,6 +74,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
 void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
 void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
 void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint);
+void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode);
 void FIO_setLiteralCompressionMode(
         FIO_prefs_t* const prefs,
         ZSTD_literalCompressionMode_e mode);
@@ -85,14 +86,14 @@ void FIO_setNotificationLevel(int level);
 *  Single File functions
 ***************************************/
 /** FIO_compressFilename() :
-    @return : 0 == ok;  1 == pb with src file. */
+ * @return : 0 == ok;  1 == pb with src file. */
 int FIO_compressFilename (FIO_prefs_t* const prefs,
                           const char* outfilename, const char* infilename,
                           const char* dictFileName, int compressionLevel,
                           ZSTD_compressionParameters comprParams);
 
 /** FIO_decompressFilename() :
-    @return : 0 == ok;  1 == pb with src file. */
+ * @return : 0 == ok;  1 == pb with src file. */
 int FIO_decompressFilename (FIO_prefs_t* const prefs,
                             const char* outfilename, const char* infilename, const char* dictFileName);
 
@@ -103,7 +104,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
 *  Multiple File functions
 ***************************************/
 /** FIO_compressMultipleFilenames() :
-    @return : nb of missing files */
+ * @return : nb of missing files */
 int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
                                   const char** inFileNamesTable, unsigned nbFiles,
                                   const char* outDirName,
@@ -112,7 +113,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
                                   ZSTD_compressionParameters comprParams);
 
 /** FIO_decompressMultipleFilenames() :
-    @return : nb of missing or skipped files */
+ * @return : nb of missing or skipped files */
 int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
                                     const char** srcNamesTable, unsigned nbFiles,
                                     const char* outDirName,
@@ -120,10 +121,12 @@ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
                                     const char* dictFileName);
 
 /* FIO_checkFilenameCollisions() :
- * Checks for and warns if thereå are any files that would have the same output path
+ * Checks for and warns if there are any files that would have the same output path
  */
 int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
 
+
+
 /*-*************************************
 *  Advanced stuff (should actually be hosted elsewhere)
 ***************************************/
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 4d1fad706..aed9ee867 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1120,7 +1120,7 @@ int main(int argCount, const char* argv[])
     }
 
 #ifndef ZSTD_NODECOMPRESS
-    if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); }  /* test mode */
+    if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); }  /* test mode */
 #endif
 
     /* No input filename ==> use stdin and stdout */

From 0a24d4ef1847cb2665507c26ec072c4dc9064969 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 16:39:47 -0700
Subject: [PATCH 083/106] removed regular file test on Windows

since it does not work well on this platform
(tested with MinGW).

Note : could be an issue within UTIL_isRegularFile()
---
 programs/fileio.c | 6 ++++++
 programs/fileio.h | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 9efb4aaed..85c04d48f 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -538,6 +538,8 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
  * @result : FILE* to `dstFileName`, or NULL if it fails */
 static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName)
 {
+    if (prefs->testMode) return NULL;  /* do not open file in test mode */
+
     assert(dstFileName != NULL);
     if (!strcmp (dstFileName, stdoutmark)) {
         DISPLAYLEVEL(4,"Using stdout for output \n");
@@ -562,10 +564,14 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
     if (UTIL_isRegularFile(dstFileName)) {
         /* Check if destination file already exists */
         FILE* const fCheck = fopen( dstFileName, "rb" );
+#if !defined(_WIN32)
+        /* this test does not work on Windows :
+         * `NUL` and `nul` are detected as regular files */
         if (!strcmp(dstFileName, nulmark)) {
             EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
                         dstFileName);
         }
+#endif
         if (fCheck != NULL) {  /* dst file exists, authorization prompt */
             fclose(fCheck);
             if (!prefs->overwrite) {
diff --git a/programs/fileio.h b/programs/fileio.h
index cb3d64c56..4b0143beb 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -26,7 +26,7 @@ extern "C" {
 #define stdinmark  "/*stdin*\\"
 #define stdoutmark "/*stdout*\\"
 #ifdef _WIN32
-#  define nulmark "nul"
+#  define nulmark "NUL"
 #else
 #  define nulmark "/dev/null"
 #endif

From caf40d0ae4e5b25bfc48921679d99935445833fe Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 16:58:49 -0700
Subject: [PATCH 084/106] fix : no output file opened in test mode

also : redistributed error code within fileio.c
for more precise diagnosis.
---
 programs/fileio.c | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 85c04d48f..948310e52 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -536,7 +536,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
 /** FIO_openDstFile() :
  *  condition : `dstFileName` must be non-NULL.
  * @result : FILE* to `dstFileName`, or NULL if it fails */
-static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName)
+static FILE*
+FIO_openDstFile(FIO_prefs_t* const prefs,
+          const char* srcFileName, const char* dstFileName)
 {
     if (prefs->testMode) return NULL;  /* do not open file in test mode */
 
@@ -936,14 +938,14 @@ FIO_compressLzmaFrame(cRess_t* ress,
     if (plain_lzma) {
         lzma_options_lzma opt_lzma;
         if (lzma_lzma_preset(&opt_lzma, compressionLevel))
-            EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName);
+            EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName);
         ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
         if (ret != LZMA_OK)
-            EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
+            EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
     } else {
         ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
         if (ret != LZMA_OK)
-            EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
+            EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
     }
 
     strm.next_in = 0;
@@ -963,11 +965,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
         ret = lzma_code(&strm, action);
 
         if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-            EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
+            EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
         {   size_t const compBytes = ress->dstBufferSize - strm.avail_out;
             if (compBytes) {
                 if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
-                    EXM_THROW(73, "Write error : %s", strerror(errno));
+                    EXM_THROW(85, "Write error : %s", strerror(errno));
                 outFileSize += compBytes;
                 strm.next_out = (BYTE*)ress->dstBuffer;
                 strm.avail_out = ress->dstBufferSize;
@@ -1657,7 +1659,7 @@ FIO_fwriteSparse(const FIO_prefs_t* const prefs,
     if (storedSkips > 1 GB) {
         int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
         if (seekResult != 0)
-            EXM_THROW(71, "1 GB skip error (sparse file support)");
+            EXM_THROW(91, "1 GB skip error (sparse file support)");
         storedSkips -= 1 GB;
     }
 
@@ -1673,13 +1675,13 @@ FIO_fwriteSparse(const FIO_prefs_t* const prefs,
 
         if (nb0T != seg0SizeT) {   /* not all 0s */
             int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
-            if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
+            if (seekResult) EXM_THROW(92, "Sparse skip error ; try --no-sparse");
             storedSkips = 0;
             seg0SizeT -= nb0T;
             ptrT += nb0T;
             {   size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
                 if (sizeCheck != seg0SizeT)
-                    EXM_THROW(73, "Write error : cannot write decoded block : %s",
+                    EXM_THROW(93, "Write error : cannot write decoded block : %s",
                             strerror(errno));
         }   }
         ptrT += seg0SizeT;
@@ -1697,11 +1699,11 @@ FIO_fwriteSparse(const FIO_prefs_t* const prefs,
             if (restPtr != restEnd) {
                 int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
                 if (seekResult)
-                    EXM_THROW(74, "Sparse skip error ; try --no-sparse");
+                    EXM_THROW(94, "Sparse skip error ; try --no-sparse");
                 storedSkips = 0;
                 {   size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
                     if (sizeCheck != (size_t)(restEnd - restPtr))
-                        EXM_THROW(75, "Write error : cannot write decoded end of block : %s",
+                        EXM_THROW(95, "Write error : cannot write decoded end of block : %s",
                             strerror(errno));
     }   }   }   }
 
@@ -2383,11 +2385,13 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
 
     if (outFileName) {
         unsigned u;
-        ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
-        if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
+        if (!prefs->testMode) {
+            ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
+            if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName);
+        }
         for (u=0; utestMode) && (fclose(ress.dstFile)))
             EXM_THROW(72, "Write error : %s : cannot properly close output file",
                         strerror(errno));
     } else {

From ad86a5d0bc8b47bbd77a70c757eef5e008850c15 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 11:15:10 -0700
Subject: [PATCH 085/106] rewrite FIO_createFilename_fromOutDir()

---
 programs/fileio.c | 62 +++++++++++++++++++++--------------------------
 tests/Makefile    |  3 ++-
 2 files changed, 30 insertions(+), 35 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 948310e52..a9075db1c 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -686,56 +686,50 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
     return 0;
 }
 
+static const char*
+extractFilename(const char* path, char separator)
+{
+    const char* search = strrchr(path, separator);
+    if (search == NULL) return path;
+    return search+1;
+}
+
 /* FIO_createFilename_fromOutDir() :
  * Takes a source file name and specified output directory, and
  * allocates memory for and returns a pointer to final path.
  * This function never returns an error (it may abort() in case of pb)
  */
 static char*
-FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, const size_t suffixLen)
+FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen)
 {
-    const char* c, *filenameBegin;
-    char* filename, *result;
-    size_t finalPathLen;
+    const char* filenameStart;
+    char separator;
+    char* result;
 
-    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
-    c = "\\";
-    #else
-    c = "/";
-    #endif
+#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
+    separator = '\\';
+#else
+    separator = '/';
+#endif
 
-    finalPathLen = strlen(outDirName);
-    filenameBegin = strrchr(srcFilename, c[0]);
-    if (filenameBegin == NULL) {
-        filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char));
-        if (!filename) {
-            EXM_THROW(30, "zstd: %s", strerror(errno));
-        }
-        strcpy(filename, srcFilename);
-    } else {
-        filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
-        if (!filename) {
-            EXM_THROW(30, "zstd: %s", strerror(errno));
-        }
-        strcpy(filename, filenameBegin+1);
-    }
+    filenameStart = extractFilename(path, separator);
+#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
+    filenameStart = extractFilename(filenameStart, '/');  /* sometimes, '/' separator is also used on Windows (mingw+msys2) */
+#endif
 
-    finalPathLen += strlen(filename);
-    result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char));
+    result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1);
     if (!result) {
-        free(filename);
-        EXM_THROW(30, "zstd: %s", strerror(errno));
+        EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno));
     }
 
-    strcpy(result, outDirName);
-    if (outDirName[strlen(outDirName)-1] == c[0]) {
-        strcat(result, filename);
+    memcpy(result, outDirName, strlen(outDirName));
+    if (outDirName[strlen(outDirName)-1] == separator) {
+        memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart));
     } else {
-        strcat(result, c);
-        strcat(result, filename);
+        memcpy(result + strlen(outDirName), &separator, 1);
+        memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart));
     }
 
-    free(filename);
     return result;
 }
 
diff --git a/tests/Makefile b/tests/Makefile
index bd2f90976..04d0b1859 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -250,7 +250,8 @@ clean:
 	$(MAKE) -C $(ZSTDDIR) clean
 	$(MAKE) -C $(PRGDIR) clean
 	@$(RM) -fR $(TESTARTEFACT)
-	@$(RM) -f core *.o tmp* *.tmp result* *.gcda dictionary *.zst \
+	@$(RM) -rf tmp*  # some test directories are named tmp*
+	@$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
         $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
         fullbench$(EXT) fullbench32$(EXT) \
         fullbench-lib$(EXT) fullbench-dll$(EXT) \

From 8c11f089a1a68914f308ef83d3fbcc607d0b8262 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 18 Oct 2019 13:34:35 -0700
Subject: [PATCH 086/106] [fuzz] Increase output buffer size of
 stream_round_trip

Fixes OSS-Fuzz crash.
Credit to OSS-Fuzz
---
 tests/fuzz/stream_round_trip.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index c534a904a..703b11713 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -125,13 +125,14 @@ static size_t compress(uint8_t *dst, size_t capacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    size_t neededBufSize;
+
     /* Give a random portion of src data to the producer, to use for
     parameter generation. The rest will be used for (de)compression */
     FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
     size = FUZZ_dataProducer_reserveDataPrefix(producer);
 
-    size_t neededBufSize;
-    neededBufSize = ZSTD_compressBound(size) * 5;
+    neededBufSize = ZSTD_compressBound(size) * 15;
 
     /* Allocate all buffers and contexts if not already allocated */
     if (neededBufSize > bufSize) {

From 29e46ed0bd1b206a3eb8c0f65a9c3399cfc6dfea Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 14:28:34 -0700
Subject: [PATCH 087/106] fix test on windows

isDirectory() doesn't work on Windows
if directory name is followed by '/'
---
 programs/util.c    | 27 +++++++++++++--------------
 tests/playTests.sh |  4 ++--
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index dbdd1cee4..2249e4513 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -328,7 +328,6 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
     unsigned i, nbFiles;
     char* buf = (char*)malloc(LIST_SIZE_INCREASE);
     char* bufend = buf + LIST_SIZE_INCREASE;
-    const char** fileTable;
 
     if (!buf) return NULL;
 
@@ -343,7 +342,7 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
                 if (!buf) return NULL;
             }
             if (buf + pos + len < bufend) {
-                memcpy(buf+pos, inputNames[i], len+1);  /* with final \0 */
+                memcpy(buf+pos, inputNames[i], len+1);  /* including final \0 */
                 pos += len + 1;
                 nbFiles++;
             }
@@ -354,20 +353,20 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
 
     if (nbFiles == 0) { free(buf); return NULL; }
 
-    fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
-    if (!fileTable) { free(buf); return NULL; }
+    {   const char** const fileTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileTable));
+        if (!fileTable) { free(buf); return NULL; }
 
-    for (i=0, pos=0; i bufend) { free(buf); free((void*)fileTable); return NULL; }
+            pos += strlen(fileTable[i]) + 1;
+        }
+
+        *allocatedBuffer = buf;
+        *allocatedNamesNb = nbFiles;
+
+        return fileTable;
     }
-
-    if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
-
-    *allocatedBuffer = buf;
-    *allocatedNamesNb = nbFiles;
-
-    return fileTable;
 }
 
 
diff --git a/tests/playTests.sh b/tests/playTests.sh
index f63a7f69e..80f36d52e 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -240,7 +240,7 @@ rm tmp         # erase source file
 touch tmp.zst  # create destination file
 $ZSTD -f tmp && die "attempt to compress a non existing file"
 test -f tmp.zst  # destination file should still be present
-rm tmp*
+rm -rf tmp*  # may also erase tmp* directory from previous failed run
 
 println "\n===> decompression only tests "
 head -c 1048576 /dev/zero > tmp
@@ -284,7 +284,7 @@ test -f tmpOutDir/tmp1.zst
 test -f tmpOutDir/tmp2.zst
 println "test : decompress multiple files into an output directory, --output-dir-flat"
 mkdir tmpOutDirDecomp
-$ZSTD tmpOutDir/ -r -d --output-dir-flat tmpOutDirDecomp
+$ZSTD tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
 test -f tmpOutDirDecomp/tmp2
 test -f tmpOutDirDecomp/tmp1
 rm -rf tmp*

From 03ef7b73a7f1d61311d747f83e9f0daa7a350488 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 14:52:16 -0700
Subject: [PATCH 088/106] attempt to run 'make check' tests on Appveyor

for mingw builds
---
 appveyor.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/appveyor.yml b/appveyor.yml
index 35f019dd6..9a3d53384 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,3 +1,7 @@
+# Following tests are run _only_ on master branch
+# To reproduce these tests, it's possible to push into a branch `appveyorTest`
+# or a branch `visual*`, they will intentionnally trigger `master` tests
+
 -
   version: 1.0.{build}
   branches:
@@ -176,6 +180,11 @@
       fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
     )
 
+
+# The following tests are for regular pushes
+# into `dev` or some feature branch
+# There run less tests, for shorter feedback loop
+
 -
   version: 1.0.{build}
   environment:
@@ -249,3 +258,11 @@
       COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
       COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
     )
+
+
+  test_script:
+  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - if [%HOST%]==[mingw] (
+      SET CC=%COMPILER%
+      make check
+    )

From 0e154fc40ea9767636ef5726806113296524df1d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 15:02:30 -0700
Subject: [PATCH 089/106] minor appveyor test script fix

---
 appveyor.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 9a3d53384..87fa5c12d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -263,6 +263,6 @@
   test_script:
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
   - if [%HOST%]==[mingw] (
-      SET CC=%COMPILER%
+      set "CC=%COMPILER%" &&
       make check
     )

From 243824551f92258c148e77dd4e6e9406c9163c21 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 18 Oct 2019 12:33:45 -0700
Subject: [PATCH 090/106] [threading] Add debug utilities

---
 lib/common/pool.c      | 10 ++++++---
 lib/common/threading.c | 47 +++++++++++++++++++++++++++++++++++++++++-
 lib/common/threading.h | 33 ++++++++++++++++++++++++++++-
 tests/poolTests.c      |  2 +-
 4 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/lib/common/pool.c b/lib/common/pool.c
index 7a8294543..f57593507 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -127,9 +127,13 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
     ctx->queueTail = 0;
     ctx->numThreadsBusy = 0;
     ctx->queueEmpty = 1;
-    (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
-    (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
-    (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
+    {
+        int error = 0;
+        error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
+        error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
+        error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
+        if (error) { POOL_free(ctx); return NULL; }
+    }
     ctx->shutdown = 0;
     /* Allocate space for the thread handles */
     ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
diff --git a/lib/common/threading.c b/lib/common/threading.c
index f3d4fa841..482664bd9 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -14,6 +14,8 @@
  * This file will hold wrapper for systems, which do not support pthreads
  */
 
+#include "threading.h"
+
 /* create fake symbol to avoid empty translation unit warning */
 int g_ZSTD_threading_useless_symbol;
 
@@ -28,7 +30,6 @@ int g_ZSTD_threading_useless_symbol;
 /* ===  Dependencies  === */
 #include 
 #include 
-#include "threading.h"
 
 
 /* ===  Implementation  === */
@@ -73,3 +74,47 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
 }
 
 #endif   /* ZSTD_MULTITHREAD */
+
+#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
+
+#include 
+
+int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
+{
+    *mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t));
+    if (!*mutex)
+        return 1;
+    return pthread_mutex_init(*mutex, attr);
+}
+
+int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
+{
+    if (!*mutex)
+        return 0;
+    {
+        int const ret = pthread_mutex_destroy(*mutex);
+        free(*mutex);
+        return ret;
+    }
+}
+
+int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
+{
+    *cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t));
+    if (!*cond)
+        return 1;
+    return pthread_cond_init(*cond, attr);
+}
+
+int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
+{
+    if (!*cond)
+        return 0;
+    {
+        int const ret = pthread_cond_destroy(*cond);
+        free(*cond);
+        return ret;
+    }
+}
+
+#endif
diff --git a/lib/common/threading.h b/lib/common/threading.h
index d806c89d0..3193ca7db 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -13,6 +13,8 @@
 #ifndef THREADING_H_938743
 #define THREADING_H_938743
 
+#include "debug.h"
+
 #if defined (__cplusplus)
 extern "C" {
 #endif
@@ -75,10 +77,12 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
  */
 
 
-#elif defined(ZSTD_MULTITHREAD)   /* posix assumed ; need a better detection method */
+#elif defined(ZSTD_MULTITHREAD)    /* posix assumed ; need a better detection method */
 /* ===   POSIX Systems   === */
 #  include 
 
+#if DEBUGLEVEL < 1
+
 #define ZSTD_pthread_mutex_t            pthread_mutex_t
 #define ZSTD_pthread_mutex_init(a, b)   pthread_mutex_init((a), (b))
 #define ZSTD_pthread_mutex_destroy(a)   pthread_mutex_destroy((a))
@@ -96,6 +100,33 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
 #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
 #define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
 
+#else /* DEBUGLEVEL >= 1 */
+
+/* Debug implementation of threading.
+ * In this implementation we use pointers for mutexes and condition variables.
+ * This way, if we forget to init/destroy them the program will crash or ASAN
+ * will report leaks.
+ */
+
+#define ZSTD_pthread_mutex_t            pthread_mutex_t*
+int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr);
+int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex);
+#define ZSTD_pthread_mutex_lock(a)      pthread_mutex_lock(*(a))
+#define ZSTD_pthread_mutex_unlock(a)    pthread_mutex_unlock(*(a))
+
+#define ZSTD_pthread_cond_t             pthread_cond_t*
+int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr);
+int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
+#define ZSTD_pthread_cond_wait(a, b)    pthread_cond_wait(*(a), *(b))
+#define ZSTD_pthread_cond_signal(a)     pthread_cond_signal(*(a))
+#define ZSTD_pthread_cond_broadcast(a)  pthread_cond_broadcast(*(a))
+
+#define ZSTD_pthread_t                  pthread_t
+#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
+#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+
+#endif
+
 #else  /* ZSTD_MULTITHREAD not defined */
 /* No multithreading support */
 
diff --git a/tests/poolTests.c b/tests/poolTests.c
index 26d57fb5c..02ec62af9 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -46,7 +46,7 @@ static int testOrder(size_t numThreads, size_t queueSize)
   POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
   ASSERT_TRUE(ctx);
   data.i = 0;
-  (void)ZSTD_pthread_mutex_init(&data.mutex, NULL);
+  ASSERT_FALSE(ZSTD_pthread_mutex_init(&data.mutex, NULL));
   { size_t i;
     for (i = 0; i < 16; ++i) {
       POOL_add(ctx, &fn, &data);

From 0bc39bc3a030d7d40e71cff0b9afe98d170d342f Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 18 Oct 2019 10:59:15 -0700
Subject: [PATCH 091/106] [zstdmt] Don't memset the jobDescription

---
 lib/compress/zstdmt_compress.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 44cbd94be..bc3062b53 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -927,12 +927,18 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
     unsigned jobID;
     DEBUGLOG(3, "ZSTDMT_releaseAllJobResources");
     for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
+        /* Copy the mutex/cond out */
+        ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex;
+        ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond;
+
         DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start);
         ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff);
-        mtctx->jobs[jobID].dstBuff = g_nullBuffer;
-        mtctx->jobs[jobID].cSize = 0;
+
+        /* Clear the job description, but keep the mutex/cond */
+        memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID]));
+        mtctx->jobs[jobID].job_mutex = mutex;
+        mtctx->jobs[jobID].job_cond = cond;
     }
-    memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
     mtctx->inBuff.buffer = g_nullBuffer;
     mtctx->inBuff.filled = 0;
     mtctx->allJobsCompleted = 1;

From 85a016ed92d7c7df1cbb2fc84db31ba4b2c36a82 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 15:21:50 -0700
Subject: [PATCH 092/106] made `make check` faster

for shorter feedback loop
---
 tests/playTests.sh | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 80f36d52e..c74d548b8 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -212,8 +212,8 @@ $ZSTD tmp -c --no-compress-literals -19      | $ZSTD -t
 $ZSTD tmp -c --compress-literals    -1       | $ZSTD -t
 $ZSTD tmp -c --compress-literals    --fast=1 | $ZSTD -t
 $ZSTD tmp -c --compress-literals    -19      | $ZSTD -t
-$ZSTD -b --fast=1 -i1e1 tmp --compress-literals
-$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals
+$ZSTD -b --fast=1 -i0e1 tmp --compress-literals
+$ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
 
 println "test : file removal"
 $ZSTD -f --rm tmp
@@ -578,30 +578,27 @@ println "- Create dictionary with short dictID"
 $ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
 println "- Create dictionaries with shrink-dict flag enabled"
-$ZSTD --train-fastcover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
-$ZSTD --train-fastcover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
-$ZSTD --train-fastcover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
+$ZSTD --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
+$ZSTD --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
+$ZSTD --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
 println "- Create dictionary with size limit"
-$ZSTD --train-fastcover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
-println "- Compare size of dictionary from 90% training samples with 80% training samples"
-$ZSTD --train-fastcover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
-$ZSTD --train-fastcover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
 println "- Create dictionary using all samples for both training and testing"
-$ZSTD --train-fastcover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
 println "- Create dictionary using f=16"
-$ZSTD --train-fastcover=f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
-$ZSTD --train-fastcover=accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
+$ZSTD --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
 println "- Create dictionary using accel=2"
-$ZSTD --train-fastcover=accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
 println "- Create dictionary using accel=10"
-$ZSTD --train-fastcover=accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
 println "- Create dictionary with multithreading"
 $ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
 println "- Test -o before --train-fastcover"
 rm -f tmpDict dictionary
-$ZSTD -o tmpDict --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
 test -f tmpDict
-$ZSTD --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
 test -f dictionary
 rm tmp* dictionary
 
@@ -675,10 +672,10 @@ $ZSTD -i0b0e3 tmp1
 println "bench negative level"
 $ZSTD -bi0 --fast tmp1
 println "with recursive and quiet modes"
-$ZSTD -rqi1b1e2 tmp1
+$ZSTD -rqi0b1e2 tmp1
 println "benchmark decompression only"
 $ZSTD -f tmp1
-$ZSTD -b -d -i1 tmp1.zst
+$ZSTD -b -d -i0 tmp1.zst
 
 println "\n===>  zstd compatibility tests "
 

From 2ad75bb574de8fbd857d220a56960e2e25b3d611 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 15:37:07 -0700
Subject: [PATCH 093/106] validated 'make test' for mingw environment

---
 tests/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 04d0b1859..161c823ed 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -265,7 +265,7 @@ clean:
 
 
 #----------------------------------------------------------------------------------
-#make valgrindTest is validated only for Linux, macOS, BSD, Hurd and Solaris targets
+# valgrind tests are validated only for some posix platforms
 #----------------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 HOST_OS = POSIX
@@ -287,7 +287,7 @@ valgrindTest: zstd datagen fuzzer fullbench
 endif
 
 
-ifneq (,$(filter MSYS%,$(shell uname)))
+ifneq (,$(filter MINGW% MSYS%,$(shell uname)))
 HOST_OS = MSYS
 endif
 

From 58c59341d34891778df3d3ce7a5606b79e764563 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 15:41:12 -0700
Subject: [PATCH 094/106] updated windows binary resources

---
 programs/windres/zstd32.res | Bin 1044 -> 1044 bytes
 programs/windres/zstd64.res | Bin 1044 -> 1044 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)

diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res
index 843499254924c1c45ed155677538d8856721f527..9984215b5430ce5ea4f91c9e12426579f5dcd0a3 100644
GIT binary patch
delta 47
tcmbQjF@(2WJ-oIC$D7A1^~*X2-W}q

delta 47
scmbQjF@(2WJ-oIC$D7A1^~*X2-W}q

delta 47
scmbQjF@
Date: Fri, 18 Oct 2019 15:45:31 -0700
Subject: [PATCH 095/106] fix function cast warning on Windows with gcc9

---
 programs/util.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index 2249e4513..c82d04e30 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -399,8 +399,8 @@ int UTIL_countPhysicalCores(void)
         DWORD returnLength = 0;
         size_t byteOffset = 0;
 
-        glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
-                                         "GetLogicalProcessorInformation");
+        glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
+                                               "GetLogicalProcessorInformation");
 
         if (glpi == NULL) {
             goto failed;

From f3796370259059657048350d22f446f5e5e2525e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 17:05:42 -0700
Subject: [PATCH 096/106] removed Visual warning

for pointer casts
---
 programs/util.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/programs/util.c b/programs/util.c
index c82d04e30..de25bc232 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -399,6 +399,11 @@ int UTIL_countPhysicalCores(void)
         DWORD returnLength = 0;
         size_t byteOffset = 0;
 
+#ifdef (_MSC_VER)
+/* Visual Studio does not like the following cast */
+#   pragma warning( disable : 4054 )  /* conversion from function ptr to data ptr */
+#   pragma warning( disable : 4055 )  /* conversion from data ptr to function ptr */
+#endif
         glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
                                                "GetLogicalProcessorInformation");
 

From 0492c570139d252eaa47cacd788317d3f66a9c6e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 17:08:52 -0700
Subject: [PATCH 097/106] fixed visual defined test

---
 programs/util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/util.c b/programs/util.c
index de25bc232..e1aa447a2 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -399,7 +399,7 @@ int UTIL_countPhysicalCores(void)
         DWORD returnLength = 0;
         size_t byteOffset = 0;
 
-#ifdef (_MSC_VER)
+#if defined(_MSC_VER)
 /* Visual Studio does not like the following cast */
 #   pragma warning( disable : 4054 )  /* conversion from function ptr to data ptr */
 #   pragma warning( disable : 4055 )  /* conversion from data ptr to function ptr */

From ca73c218be4793acef114ad6154cca772b3540c5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 17:22:45 -0700
Subject: [PATCH 098/106] added mode documentation on ZSTD_CLEVEL

providing range restriction [1-19] explicitly
partially answering #1829
---
 programs/README.md  |  9 ++++++---
 programs/zstd.1     |  5 ++++-
 programs/zstd.1.md  | 10 ++++++++++
 programs/zstdgrep.1 |  2 +-
 programs/zstdless.1 |  2 +-
 5 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/programs/README.md b/programs/README.md
index c3a5590d6..7668d49a2 100644
--- a/programs/README.md
+++ b/programs/README.md
@@ -173,10 +173,13 @@ Benchmark arguments :
 ```
 
 #### Restricted usage of Environment Variables
-Using environment variables to set compression/decompression parameters has security implications. Therefore,
-we intentionally restrict its usage. Currently, only `ZSTD_CLEVEL` is supported for setting compression level.
+Using environment variables to set parameters has security implications.
+Therefore, this avenue is intentionally restricted.
+Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
+`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
 If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
-Note that command line options will override corresponding environment variable settings.
+`ZSTD_CLEVEL` just replaces the default compression level (`3`).
+It can be overridden by corresponding command line arguments.
 
 #### Long distance matching mode
 The long distance matching mode, enabled with `--long`, is designed to improve
diff --git a/programs/zstd.1 b/programs/zstd.1
index bb5103c61..1072c3233 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "September 2019" "zstd 1.4.4" "User Commands"
+.TH "ZSTD" "1" "October 2019" "zstd 1.4.4" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -206,6 +206,9 @@ add integrity check computed from uncompressed data (default: enabled)
 \fB\-\-\fR
 All arguments after \fB\-\-\fR are treated as files
 .
+.SS "Restricted usage of Environment Variables"
+Using environment variables to set parameters has security implications\. Therefore, this avenue is intentionally restricted\. Only \fBZSTD_CLEVEL\fR is supported currently, for setting compression level\. \fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\. It can be overridden by corresponding command line arguments\.
+.
 .SH "DICTIONARY BUILDER"
 \fBzstd\fR offers \fIdictionary\fR compression, which greatly improves efficiency on small files and messages\. It\'s possible to train \fBzstd\fR with a set of samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary, using command \fB\-D dictionaryFileName\fR\. Compression of small files similar to the sample set will be greatly improved\.
 .
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index dff4d9eac..c1f321bc0 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -214,6 +214,16 @@ the last one takes effect.
 * `--`:
     All arguments after `--` are treated as files
 
+### Restricted usage of Environment Variables
+
+Using environment variables to set parameters has security implications.
+Therefore, this avenue is intentionally restricted.
+Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
+`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
+If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
+`ZSTD_CLEVEL` just replaces the default compression level (`3`).
+It can be overridden by corresponding command line arguments.
+
 
 DICTIONARY BUILDER
 ------------------
diff --git a/programs/zstdgrep.1 b/programs/zstdgrep.1
index 06927ab78..b97f8cabc 100644
--- a/programs/zstdgrep.1
+++ b/programs/zstdgrep.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTDGREP" "1" "September 2019" "zstd 1.4.4" "User Commands"
+.TH "ZSTDGREP" "1" "October 2019" "zstd 1.4.4" "User Commands"
 .
 .SH "NAME"
 \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
diff --git a/programs/zstdless.1 b/programs/zstdless.1
index d49042276..1ecc8bdc5 100644
--- a/programs/zstdless.1
+++ b/programs/zstdless.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTDLESS" "1" "September 2019" "zstd 1.4.4" "User Commands"
+.TH "ZSTDLESS" "1" "October 2019" "zstd 1.4.4" "User Commands"
 .
 .SH "NAME"
 \fBzstdless\fR \- view zstandard\-compressed files

From ff7bd16c0a3676e3d87063a3355eff0c9cc4f090 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 17:48:12 -0700
Subject: [PATCH 099/106] clarifications for the FSE decoding table

requested in #1782
---
 doc/zstd_compression_format.md | 58 ++++++++++++++++++----------------
 1 file changed, 30 insertions(+), 28 deletions(-)

diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index 111dd98ae..90ac0fe9b 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.3.3 (16/08/19)
+0.3.4 (16/08/19)
 
 
 Introduction
@@ -1107,18 +1107,18 @@ It follows the following build rule :
 
 The table has a size of `Table_Size = 1 << Accuracy_Log`.
 Each cell describes the symbol decoded,
-and instructions to get the next state.
+and instructions to get the next state (`Number_of_Bits` and `Baseline`).
 
 Symbols are scanned in their natural order for "less than 1" probabilities.
 Symbols with this probability are being attributed a single cell,
 starting from the end of the table and retreating.
 These symbols define a full state reset, reading `Accuracy_Log` bits.
 
-All remaining symbols are allocated in their natural order.
-Starting from symbol `0` and table position `0`,
+Then, all remaining symbols, sorted in natural order, are allocated cells.
+Starting from symbol `0` (if it exists), and table position `0`,
 each symbol gets allocated as many cells as its probability.
 Cell allocation is spreaded, not linear :
-each successor position follow this rule :
+each successor position follows this rule :
 
 ```
 position += (tableSize>>1) + (tableSize>>3) + 3;
@@ -1130,40 +1130,41 @@ A position is skipped if already occupied by a "less than 1" probability symbol.
 each position in the table, switching to the next symbol when enough
 states have been allocated to the current one.
 
-The result is a list of state values.
-Each state will decode the current symbol.
+The process guarantees that the table is entirely filled.
+Each cell corresponds to a state value, which contains the symbol being decoded.
 
-To get the `Number_of_Bits` and `Baseline` required for next state,
-it's first necessary to sort all states in their natural order.
-The lower states will need 1 more bit than higher ones.
+To add the `Number_of_Bits` and `Baseline` required to retrieve next state,
+it's first necessary to sort all occurrences of each symbol in state order.
+Lower states will need 1 more bit than higher ones.
 The process is repeated for each symbol.
 
 __Example__ :
-Presuming a symbol has a probability of 5.
-It receives 5 state values. States are sorted in natural order.
+Presuming a symbol has a probability of 5,
+it receives 5 cells, corresponding to 5 state values.
+These state values are then sorted in natural order.
 
-Next power of 2 is 8.
-Space of probabilities is divided into 8 equal parts.
-Presuming the `Accuracy_Log` is 7, it defines 128 states.
+Next power of 2 after 5 is 8.
+Space of probabilities must be divided into 8 equal parts.
+Presuming the `Accuracy_Log` is 7, it defines a space of 128 states.
 Divided by 8, each share is 16 large.
 
-In order to reach 8, 8-5=3 lowest states will count "double",
-doubling the number of shares (32 in width),
-requiring one more bit in the process.
+In order to reach 8 shares, 8-5=3 lowest states will count "double",
+doubling their shares (32 in width), hence requiring one more bit.
 
 Baseline is assigned starting from the higher states using fewer bits,
-and proceeding naturally, then resuming at the first state,
-each takes its allocated width from Baseline.
+increasing at each state, then resuming at the first state,
+each state takes its allocated width from Baseline.
 
-| state order      |   0   |   1   |    2   |   3  |   4   |
-| ---------------- | ----- | ----- | ------ | ---- | ----- |
-| width            |  32   |  32   |   32   |  16  |  16   |
-| `Number_of_Bits` |   5   |   5   |    5   |   4  |   4   |
-| range number     |   2   |   4   |    6   |   0  |   1   |
-| `Baseline`       |  32   |  64   |   96   |   0  |  16   |
-| range            | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
+| state value      |   1   |  39   |   77   |  84  |  122   |
+| state order      |   0   |   1   |    2   |   3  |    4   |
+| ---------------- | ----- | ----- | ------ | ---- | ------ |
+| width            |  32   |  32   |   32   |  16  |   16   |
+| `Number_of_Bits` |   5   |   5   |    5   |   4  |    4   |
+| range number     |   2   |   4   |    6   |   0  |    1   |
+| `Baseline`       |  32   |  64   |   96   |   0  |   16   |
+| range            | 32-63 | 64-95 | 96-127 | 0-15 | 16-31  |
 
-The next state is determined from current state
+During decoding, the next state value is determined from current state value,
 by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
 
 See [Appendix A] for the results of this process applied to the default distributions.
@@ -1657,6 +1658,7 @@ or at least provide a meaningful error code explaining for which reason it canno
 
 Version changes
 ---------------
+- 0.3.4 : clarifications for FSE decoding table
 - 0.3.3 : clarifications for field Block_Size
 - 0.3.2 : remove additional block size restriction on compressed blocks
 - 0.3.1 : minor clarification regarding offset history update rules

From 632e07747c1a4ebf419f1b84b7b2358e5987ce26 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 21 Oct 2019 12:14:59 -0700
Subject: [PATCH 100/106] minor refactor of FIO_compressGzFrame()

mostly narrowing variables lifetime and mutability.
---
 programs/fileio.c | 54 +++++++++++++++++++++++------------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index a9075db1c..f4384484c 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -834,13 +834,12 @@ static void FIO_freeCResources(cRess_t ress)
 
 #ifdef ZSTD_GZCOMPRESS
 static unsigned long long
-FIO_compressGzFrame(cRess_t* ress,
+FIO_compressGzFrame(const cRess_t* ress,  /* buffers & handlers are used, but not changed */
                     const char* srcFileName, U64 const srcFileSize,
                     int compressionLevel, U64* readsize)
 {
     unsigned long long inFileSize = 0, outFileSize = 0;
     z_stream strm;
-    int ret;
 
     if (compressionLevel > Z_BEST_COMPRESSION)
         compressionLevel = Z_BEST_COMPRESSION;
@@ -849,11 +848,12 @@ FIO_compressGzFrame(cRess_t* ress,
     strm.zfree = Z_NULL;
     strm.opaque = Z_NULL;
 
-    ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
+    {   int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
                         15 /* maxWindowLogSize */ + 16 /* gzip only */,
                         8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
-    if (ret != Z_OK)
-        EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
+        if (ret != Z_OK) {
+            EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
+    }   }
 
     strm.next_in = 0;
     strm.avail_in = 0;
@@ -861,6 +861,7 @@ FIO_compressGzFrame(cRess_t* ress,
     strm.avail_out = (uInt)ress->dstBufferSize;
 
     while (1) {
+        int ret;
         if (strm.avail_in == 0) {
             size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
             if (inSize == 0) break;
@@ -871,32 +872,31 @@ FIO_compressGzFrame(cRess_t* ress,
         ret = deflate(&strm, Z_NO_FLUSH);
         if (ret != Z_OK)
             EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret);
-        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
-            if (decompBytes) {
-                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
-                    EXM_THROW(73, "Write error : cannot write to output file : %s", strerror(errno));
-                outFileSize += decompBytes;
+        {   size_t const cSize = ress->dstBufferSize - strm.avail_out;
+            if (cSize) {
+                if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
+                    EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno));
+                outFileSize += cSize;
                 strm.next_out = (Bytef*)ress->dstBuffer;
                 strm.avail_out = (uInt)ress->dstBufferSize;
-            }
-        }
-        if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
-            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
+        }   }
+        if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
                             (unsigned)(inFileSize>>20),
                             (double)outFileSize/inFileSize*100)
-        else
-            DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
+        } else {
+            DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
                             (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
                             (double)outFileSize/inFileSize*100);
-    }
+    }   }
 
     while (1) {
-        ret = deflate(&strm, Z_FINISH);
-        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
-            if (decompBytes) {
-                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
-                    EXM_THROW(75, "Write error : %s", strerror(errno));
-                outFileSize += decompBytes;
+        int const ret = deflate(&strm, Z_FINISH);
+        {   size_t const cSize = ress->dstBufferSize - strm.avail_out;
+            if (cSize) {
+                if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
+                    EXM_THROW(75, "Write error : %s ", strerror(errno));
+                outFileSize += cSize;
                 strm.next_out = (Bytef*)ress->dstBuffer;
                 strm.avail_out = (uInt)ress->dstBufferSize;
         }   }
@@ -905,11 +905,11 @@ FIO_compressGzFrame(cRess_t* ress,
             EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret);
     }
 
-    ret = deflateEnd(&strm);
-    if (ret != Z_OK)
-        EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
+    {   int const ret = deflateEnd(&strm);
+        if (ret != Z_OK) {
+            EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
+    }   }
     *readsize = inFileSize;
-
     return outFileSize;
 }
 #endif

From bad35bd307fd9dafa1c675e56ed3abafb06af522 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 21 Oct 2019 12:55:39 -0700
Subject: [PATCH 101/106] turned off zlib tests during msan

since the local zlib library is not msan-instrumented
---
 .travis.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 20fa9beeb..a6e1a99ea 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -54,7 +54,10 @@ matrix:
     - name: Trusty (clang-3.8 + MSan + Test Zstd)
       script:
         - make clang38install
-        - CC=clang-3.8 make clean msan-test-zstd
+        # External libraries must be turned off when using MSAN tests,
+        # because they are not msan-instrumented,
+        # so any data coming from these libraries is always considered "uninitialized"
+        - CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
 
     - name: Trusty (Minimal Decompressor Macros)
       script:

From 989e0f0cee41727a76087a323dadc93561297250 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 21 Oct 2019 14:33:50 -0700
Subject: [PATCH 102/106] Added --output-dir-flat=

which is a preferred construction when applicable.

Also, added --output-dir-flat into man page.
---
 programs/zstd.1    | 4 ++++
 programs/zstd.1.md | 7 +++++++
 programs/zstdcli.c | 7 ++++---
 tests/playTests.sh | 4 ++++
 4 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/programs/zstd.1 b/programs/zstd.1
index 1072c3233..fef0e76e0 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -175,6 +175,10 @@ keep source file(s) after successful compression or decompression\. This is the
 operate recursively on directories
 .
 .TP
+\fB\-\-output\-dir\-flat[=dir]\fR
+resulting files are stored into target \fBdir\fR directory, instead of same directory as origin file\. Be aware that this command can introduce name collision issues, if multiple files, from different directories, end up having the same name\. Collision resolution ensures first file with a given name will be present in \fBdir\fR, while in combination with \fB\-f\fR, the last file will be present instead\.
+.
+.TP
 \fB\-\-format=FORMAT\fR
 compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBzstd\fR, \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\. If no such format is provided, \fBzstd\fR is the default\.
 .
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index c1f321bc0..e3daa4c87 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -191,6 +191,13 @@ the last one takes effect.
     This is the default behavior.
 * `-r`:
     operate recursively on directories
+* `--output-dir-flat[=dir]`:
+    resulting files are stored into target `dir` directory,
+    instead of same directory as origin file.
+    Be aware that this command can introduce name collision issues,
+    if multiple files, from different directories, end up having the same name.
+    Collision resolution ensures first file with a given name will be present in `dir`,
+    while in combination with `-f`, the last file will be present instead.
 * `--format=FORMAT`:
     compress and decompress in other formats. If compiled with
     support, zstd can compress to or decompress from other compression algorithm
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index aed9ee867..803c50377 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -136,11 +136,10 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
     DISPLAY( " -l     : print information about zstd compressed files \n");
-    DISPLAY( " --output-dir-flat directory: results stored into `directory`. Filename collisions mean first file will be compressed. With -f, the last file will be compressed.\n");
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
     DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
-    DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
+    DISPLAY( "--fast[=#]: switch to very fast compression levels (default: %u)\n", 1);
     DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n");
     DISPLAY( "--stream-size=# : optimize compression parameters for streaming input of given number of bytes \n");
     DISPLAY( "--size-hint=# optimize compression parameters for streaming input of approximately this size\n");
@@ -148,7 +147,7 @@ static int usage_advanced(const char* programName)
 #ifdef ZSTD_MULTITHREAD
     DISPLAY( " -T#    : spawns # compression threads (default: 1, 0==# cores) \n");
     DISPLAY( " -B#    : select size of each job (default: 0==automatic) \n");
-    DISPLAY( " --rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
+    DISPLAY( "--rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
 #endif
     DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
     DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
@@ -156,6 +155,7 @@ static int usage_advanced(const char* programName)
 #endif
 #ifdef UTIL_HAS_CREATEFILELIST
     DISPLAY( " -r     : operate recursively on directories \n");
+    DISPLAY( "--output-dir-flat[=directory]: all resulting files stored into `directory`. \n");
 #endif
     DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
 #ifdef ZSTD_GZCOMPRESS
@@ -756,6 +756,7 @@ int main(int argCount, const char* argv[])
                     if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readU32FromChar(&argument); continue; }
+                    if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; }
                     if (longCommandWArg(&argument, "--long")) {
                         unsigned ldmWindowLog = 0;
                         ldmFlag = 1;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index c74d548b8..796a5bde9 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -287,6 +287,10 @@ mkdir tmpOutDirDecomp
 $ZSTD tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
 test -f tmpOutDirDecomp/tmp2
 test -f tmpOutDirDecomp/tmp1
+rm -f tmpOutDirDecomp/*
+$ZSTD tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp
+test -f tmpOutDirDecomp/tmp2
+test -f tmpOutDirDecomp/tmp1
 rm -rf tmp*
 
 println "\n===>  Advanced compression parameters "

From b1ec94e63cfb4ba75a7268d15ec706ed0dcb8ce2 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 21 Oct 2019 19:42:14 -0700
Subject: [PATCH 103/106] Fix ZSTD_f_zstd1_magicless for small data

* Fix `ZSTD_FRAMEHEADERSIZE_PREFIX` and `ZSTD_FRAMEHEADERSIZE_MIN` to
  take a `format` parameter, so it is impossible to get the wrong size.
* Fix the places that called `ZSTD_FRAMEHEADERSIZE_PREFIX` without
  taking the format into account, which is now impossible by design.
* Call `ZSTD_frameHeaderSize_internal()` with `dctx->format`.
* The added tests catch both bugs in `ZSTD_decompressFrame()`.

Fixes #1813.
---
 lib/decompress/zstd_decompress.c | 26 ++++++++++++--------------
 lib/zstd.h                       |  4 ++--
 programs/fileio.c                |  2 +-
 tests/fullbench.c                |  6 +++---
 tests/fuzzer.c                   | 30 ++++++++++++++++++++++++++++++
 zlibWrapper/zstd_zlibwrapper.c   |  4 ++--
 6 files changed, 50 insertions(+), 22 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 751060b2c..ca47a6678 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -88,10 +88,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
 
 static size_t ZSTD_startingInputLength(ZSTD_format_e format)
 {
-    size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
-                    ZSTD_FRAMEHEADERSIZE_PREFIX - ZSTD_FRAMEIDSIZE :
-                    ZSTD_FRAMEHEADERSIZE_PREFIX;
-    ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
+    size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format);
     /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
     assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) );
     return startingInputLength;
@@ -376,7 +373,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
 {
     unsigned long long totalDstSize = 0;
 
-    while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
+    while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) {
         U32 const magicNumber = MEM_readLE32(src);
 
         if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
@@ -629,11 +626,12 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
 
     /* check */
     RETURN_ERROR_IF(
-        remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize,
+        remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize,
         srcSize_wrong);
 
     /* Frame Header */
-    {   size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX);
+    {   size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal(
+                ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format);
         if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
         RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
                         srcSize_wrong);
@@ -714,7 +712,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
         dictSize = ZSTD_DDict_dictSize(ddict);
     }
 
-    while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
+    while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
         if (ZSTD_isLegacy(src, srcSize)) {
@@ -1300,14 +1298,14 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz
 
 
 /* ZSTD_initDStream_usingDict() :
- * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX.
+ * return : expected size, aka ZSTD_startingInputLength().
  * this function cannot fail */
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
 {
     DEBUGLOG(4, "ZSTD_initDStream_usingDict");
     FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) );
     FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
-    return ZSTD_FRAMEHEADERSIZE_PREFIX;
+    return ZSTD_startingInputLength(zds->format);
 }
 
 /* note : this variant can't fail */
@@ -1324,16 +1322,16 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
 {
     FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) );
     FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) );
-    return ZSTD_FRAMEHEADERSIZE_PREFIX;
+    return ZSTD_startingInputLength(dctx->format);
 }
 
 /* ZSTD_resetDStream() :
- * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX.
+ * return : expected size, aka ZSTD_startingInputLength().
  * this function cannot fail */
 size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
 {
     FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only));
-    return ZSTD_FRAMEHEADERSIZE_PREFIX;
+    return ZSTD_startingInputLength(dctx->format);
 }
 
 
@@ -1564,7 +1562,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                             zds->lhSize += remainingInput;
                         }
                         input->pos = input->size;
-                        return (MAX(ZSTD_FRAMEHEADERSIZE_MIN, hSize) - zds->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
+                        return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
                     }
                     assert(ip != NULL);
                     memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
diff --git a/lib/zstd.h b/lib/zstd.h
index c57f2debc..22711d17b 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1017,8 +1017,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  * Some of them might be removed in the future (especially when redundant with existing stable functions)
  * ***************************************************************************************/
 
-#define ZSTD_FRAMEHEADERSIZE_PREFIX 5   /* minimum input size required to query frame header size */
-#define ZSTD_FRAMEHEADERSIZE_MIN    6
+#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1)   /* minimum input size required to query frame header size */
+#define ZSTD_FRAMEHEADERSIZE_MIN(format)    ((format) == ZSTD_f_zstd1 ? 6 : 2)
 #define ZSTD_FRAMEHEADERSIZE_MAX   18   /* can be useful for static allocation */
 #define ZSTD_SKIPPABLEHEADERSIZE    8
 
diff --git a/programs/fileio.c b/programs/fileio.c
index f4384484c..16d06287d 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -2443,7 +2443,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
     for ( ; ; ) {
         BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
         size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
-        if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN) {
+        if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)) {
             if ( feof(srcFile)
               && (numBytesRead == 0)
               && (info->compressedSize > 0)
diff --git a/tests/fullbench.c b/tests/fullbench.c
index f750ee0d7..0e2761e11 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -450,7 +450,7 @@ static int benchMem(unsigned benchNb,
     case 31:  /* ZSTD_decodeLiteralsBlock : starts literals block in dstBuff2 */
         {   size_t frameHeaderSize;
             g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
-            frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX);
+            frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
             CONTROL(!ZSTD_isError(frameHeaderSize));
             /* check block is compressible, hence contains a literals section */
             {   blockProperties_t bp;
@@ -471,10 +471,10 @@ static int benchMem(unsigned benchNb,
             const BYTE* ip = dstBuff;
             const BYTE* iend;
             {   size_t const cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
-                CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX);
+                CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
             }
             /* Skip frame Header */
-            {   size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX);
+            {   size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
                 CONTROL(!ZSTD_isError(frameHeaderSize));
                 ip += frameHeaderSize;
             }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a109a440d..88f3b83f8 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1886,6 +1886,36 @@ static int basicUnitTests(U32 const seed, double compressibility)
             DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
         }
 
+        /* basic block compression */
+        DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
+        CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
+        {   ZSTD_inBuffer in = { CNBuffer, 0, 0 };
+            ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
+            size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
+            if (result != 0) goto _output_error;
+            if (in.pos != in.size) goto _output_error;
+            cSize = out.pos;
+        }
+        DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
+
+        DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
+        ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
+        CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
+        /* one shot */
+        {   size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+            if (result != 0) goto _output_error;
+            DISPLAYLEVEL(3, "one-shot OK, ");
+        }
+        /* streaming */
+        {   ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
+            ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
+            size_t const result = ZSTD_decompressStream(dctx, &out, &in);
+            if (result != 0) goto _output_error;
+            if (in.pos != in.size) goto _output_error;
+            if (out.pos != 0) goto _output_error;
+            DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
+        }
+
         ZSTD_freeCCtx(cctx);
         ZSTD_freeDCtx(dctx);
     }
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index cb6afa786..3fa442106 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -31,7 +31,7 @@
 /* ===   Constants   === */
 #define Z_INFLATE_SYNC              8
 #define ZLIB_HEADERSIZE             4
-#define ZSTD_HEADERSIZE             ZSTD_FRAMEHEADERSIZE_MIN
+#define ZSTD_HEADERSIZE             ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)
 #define ZWRAP_DEFAULT_CLEVEL        3   /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
 
 
@@ -457,7 +457,7 @@ static void ZWRAP_initDCtx(ZWRAP_DCtx* zwd)
 static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
 {
     ZWRAP_DCtx* zwd;
-    MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN);   /* check static buffer size condition */
+    MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_HEADERSIZE);   /* check static buffer size condition */
 
     if (strm->zalloc && strm->zfree) {
         zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));

From 111b0c53b02417a13f1020ef413bfe49bd6c518f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 22 Oct 2019 13:51:18 -0700
Subject: [PATCH 104/106] update documentation on deprecated functions

mostly : note that these functions will soon generate deprecation warnings
---
 doc/zstd_manual.html | 236 ++++++++++++++++++++++++-------------------
 lib/zstd.h           | 105 ++++++++++++-------
 2 files changed, 201 insertions(+), 140 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 4a0d3ee37..0021eec28 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -27,10 +27,16 @@
 
  • Advanced compression functions
    • 
 
  • Advanced decompression functions
    • 
 
  • Advanced streaming functions
    • 
-
  • Buffer-less and synchronous inner streaming functions
    • 
-
  • Buffer-less streaming compression (synchronous mode)
    • 
-
  • Buffer-less streaming decompression (synchronous mode)
    • 
-
  • Block level API
    • 
+
  • ! ZSTD_initCStream_usingDict() :
    • 
+
  • ! ZSTD_initCStream_advanced() :
    • 
+
  • ! ZSTD_initCStream_usingCDict() :
    • 
+
  • ! ZSTD_initCStream_usingCDict_advanced() :
    • 
+
  • This function is deprecated, and is equivalent to:
    • 
+
  • This function is deprecated, and is equivalent to:
    • 
+
  • Buffer-less and synchronous inner streaming functions
    • 
+
  • Buffer-less streaming compression (synchronous mode)
    • 
+
  • Buffer-less streaming decompression (synchronous mode)
    • 
+
  • Block level API
    • 
 
 
    
 
    Introduction
    @@ -157,9 +163,13 @@ size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
                          void* dst, size_t dstCapacity,
                    const void* src, size_t srcSize,
                          int compressionLevel);
-
      Same as ZSTD_compress(), using an explicit ZSTD_CCtx
-  The function will compress at requested compression level,
-  ignoring any other parameter 
+
      Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
+  Important : in order to behave similarly to `ZSTD_compress()`,
+  this function compresses at requested compression level,
+  __ignoring any other parameter__ .
+  If any advanced parameter was set using the advanced API,
+  they will all be reset. Only `compressionLevel` remains.
+ 
 

    
 
 
    Decompression context
      When decompressing many times,
@@ -199,18 +209,26 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
     /* compression parameters
      * Note: When compressing with a ZSTD_CDict these parameters are superseded
-     * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
-     * for more info (superseded-by-cdict). */
-    ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
+     * by the parameters used to construct the ZSTD_CDict.
+     * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
+    ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
+                              * Note that exact compression parameters are dynamically determined,
+                              * depending on both compression level and srcSize (when known).
                               * Default level is ZSTD_CLEVEL_DEFAULT==3.
                               * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
                               * Note 1 : it's possible to pass a negative compression level.
-                              * Note 2 : setting a level sets all default values of other compression parameters */
+                              * Note 2 : setting a level resets all other compression parameters to default */
+    /* Advanced compression parameters :
+     * It's possible to pin down compression parameters to some specific values.
+     * In which case, these values are no longer dynamically selected by the compressor */
     ZSTD_c_windowLog=101,    /* Maximum allowed back-reference distance, expressed as power of 2.
+                              * This will set a memory budget for streaming decompression,
+                              * with larger values requiring more memory
+                              * and typically compressing more.
                               * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
                               * Special: value 0 means "use default windowLog".
                               * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
-                              *       requires explicitly allowing such window size at decompression stage if using streaming. */
+                              *       requires explicitly allowing such size at streaming decompression stage. */
     ZSTD_c_hashLog=102,      /* Size of the initial probe table, as a power of 2.
                               * Resulting memory usage is (1 << (hashLog+2)).
                               * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@@ -221,13 +239,13 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                               * Resulting memory usage is (1 << (chainLog+2)).
                               * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
                               * Larger tables result in better and slower compression.
-                              * This parameter is useless when using "fast" strategy.
+                              * This parameter is useless for "fast" strategy.
                               * It's still useful when using "dfast" strategy,
                               * in which case it defines a secondary probe table.
                               * Special: value 0 means "use default chainLog". */
     ZSTD_c_searchLog=104,    /* Number of search attempts, as a power of 2.
                               * More attempts result in better and slower compression.
-                              * This parameter is useless when using "fast" and "dFast" strategies.
+                              * This parameter is useless for "fast" and "dFast" strategies.
                               * Special: value 0 means "use default searchLog". */
     ZSTD_c_minMatch=105,     /* Minimum size of searched matches.
                               * Note that Zstandard can still find matches of smaller size,
@@ -282,7 +300,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
     ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
                               * Content size must be known at the beginning of compression.
                               * This is automatically the case when using ZSTD_compress2(),
-                              * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
+                              * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
     ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_c_dictIDFlag=202,   /* When applicable, dictionary's ID is written into frame header (default:1) */
 
@@ -301,7 +319,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                               * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
                               * 0 means default, which is dynamically determined based on compression parameters.
                               * Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
-                              * The minimum size is automatically and transparently enforced */
+                              * The minimum size is automatically and transparently enforced. */
     ZSTD_c_overlapLog=402,   /* Control the overlap size, as a fraction of window size.
                               * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
                               * It helps preserve compression ratio, while each job is compressed in parallel.
@@ -335,7 +353,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
      ZSTD_c_experimentalParam4=1001,
      ZSTD_c_experimentalParam5=1002,
      ZSTD_c_experimentalParam6=1003,
-     ZSTD_c_experimentalParam7=1004,
+     ZSTD_c_experimentalParam7=1004
 } ZSTD_cParameter;
 

    
 
    typedef struct {
@@ -913,21 +931,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 

    
 
    typedef enum {
     ZSTD_dlm_byCopy = 0,  /**< Copy dictionary content internally */
-    ZSTD_dlm_byRef = 1,   /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
+    ZSTD_dlm_byRef = 1    /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
 } ZSTD_dictLoadMethod_e;
 

    
 
    typedef enum {
-    /* Opened question : should we have a format ZSTD_f_auto ?
-     * Today, it would mean exactly the same as ZSTD_f_zstd1.
-     * But, in the future, should several formats become supported,
-     * on the compression side, it would mean "default format".
-     * On the decompression side, it would mean "automatic format detection",
-     * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
-     * Since meaning is a little different, another option could be to define different enums for compression and decompression.
-     * This question could be kept for later, when there are actually multiple formats to support,
-     * but there is also the question of pinning enum values, and pinning value `0` is especially important */
     ZSTD_f_zstd1 = 0,           /* zstd frame format, specified in zstd_compression_format.md (default) */
-    ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number.
+    ZSTD_f_zstd1_magicless = 1  /* Variant of zstd frame format, without initial 4-bytes magic number.
                                  * Useful to save 4 bytes per generated frame.
                                  * Decoder cannot recognise automatically this format, requiring this instruction. */
 } ZSTD_format_e;
@@ -961,7 +970,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
      */
     ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
     ZSTD_dictForceAttach   = 1, /* Never copy the dictionary. */
-    ZSTD_dictForceCopy     = 2, /* Always copy the dictionary. */
+    ZSTD_dictForceCopy     = 2  /* Always copy the dictionary. */
 } ZSTD_dictAttachPref_e;
 

    
 
    typedef enum {
@@ -970,7 +979,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
                                *   levels will be compressed. */
   ZSTD_lcm_huffman = 1,       /**< Always attempt Huffman compression. Uncompressed literals will still be
                                *   emitted if Huffman compression is not profitable. */
-  ZSTD_lcm_uncompressed = 2,  /**< Always emit uncompressed literals. */
+  ZSTD_lcm_uncompressed = 2   /**< Always emit uncompressed literals. */
 } ZSTD_literalCompressionMode_e;
 

    
 
    Frame size functions
    
@@ -1123,7 +1132,8 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
 
      Create a digested dictionary for compression
   Dictionary content is just referenced, not duplicated.
   As a consequence, `dictBuffer` **must** outlive CDict,
-  and its content must remain unmodified throughout the lifetime of CDict. 
+  and its content must remain unmodified throughout the lifetime of CDict.
+  note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef 
 

    
 
 
    ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
@@ -1154,7 +1164,9 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
                         const void* src, size_t srcSize,
                         const void* dict,size_t dictSize,
                               ZSTD_parameters params);
-
      Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) 
+
      Note : this function is now DEPRECATED.
+         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
+  This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x 
 

    
 
 
    size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
@@ -1162,7 +1174,9 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  /**< t
                             const void* src, size_t srcSize,
                             const ZSTD_CDict* cdict,
                                   ZSTD_frameParameters fParams);
-
      Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters 
+
      Note : this function is now REDUNDANT.
+         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
+  This prototype will be marked as deprecated and generate compilation warning in some future version 
 

    
 
 
    size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
@@ -1339,59 +1353,67 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
  * pledgedSrcSize must be correct. If it is not known at init time, use
  * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
  * "0" also disables frame content size field. It may be enabled in the future.
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
-size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
-/**! ZSTD_initCStream_usingDict() :
- * This function is deprecated, and is equivalent to:
- *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
- *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
- *
- * Creates of an internal CDict (incompatible with static CCtx), except if
- * dict == NULL or dictSize < 8, in which case no dict is used.
- * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
- * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
- */
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
-/**! ZSTD_initCStream_advanced() :
- * This function is deprecated, and is approximately equivalent to:
- *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     // Pseudocode: Set each zstd parameter and leave the rest as-is.
- *     for ((param, value) : params) {
- *         ZSTD_CCtx_setParameter(zcs, param, value);
- *     }
- *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
- *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
- *
- * pledgedSrcSize must be correct. If srcSize is not known at init time, use
- * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
- */
-size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);
-/**! ZSTD_initCStream_usingCDict() :
- * This function is deprecated, and equivalent to:
- *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     ZSTD_CCtx_refCDict(zcs, cdict);
- *
- * note : cdict will just be referenced, and must outlive compression session
- */
-size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
-/**! ZSTD_initCStream_usingCDict_advanced() :
- * This function is deprecated, and is approximately equivalent to:
- *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
- *     for ((fParam, value) : fParams) {
- *         ZSTD_CCtx_setParameter(zcs, fParam, value);
- *     }
- *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
- *     ZSTD_CCtx_refCDict(zcs, cdict);
- *
- * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
- * pledgedSrcSize must be correct. If srcSize is not known at init time, use
- * value ZSTD_CONTENTSIZE_UNKNOWN.
- */
-size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
+size_t
+ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
+                         int compressionLevel,
+                         unsigned long long pledgedSrcSize);
 

    
+
    ! ZSTD_initCStream_usingDict() :
     This function is deprecated, and is equivalent to:
+     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+
+ Creates of an internal CDict (incompatible with static CCtx), except if
+ dict == NULL or dictSize < 8, in which case no dict is used.
+ Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
+ it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ 
+
    
+
+
    ! ZSTD_initCStream_advanced() :
     This function is deprecated, and is approximately equivalent to:
+     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+     // Pseudocode: Set each zstd parameter and leave the rest as-is.
+     for ((param, value) : params) {
+         ZSTD_CCtx_setParameter(zcs, param, value);
+     }
+     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
+
+ dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
+ pledgedSrcSize must be correct.
+ If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ 
+
    
+
+
    ! ZSTD_initCStream_usingCDict() :
     This function is deprecated, and equivalent to:
+     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+     ZSTD_CCtx_refCDict(zcs, cdict);
+
+ note : cdict will just be referenced, and must outlive compression session
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ 
+
    
+
+
    ! ZSTD_initCStream_usingCDict_advanced() :
       This function is DEPRECATED, and is approximately equivalent to:
+     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
+     // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
+     for ((fParam, value) : fParams) {
+         ZSTD_CCtx_setParameter(zcs, fParam, value);
+     }
+     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
+     ZSTD_CCtx_refCDict(zcs, cdict);
+
+ same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
+ pledgedSrcSize must be correct. If srcSize is not known at init time, use
+ value ZSTD_CONTENTSIZE_UNKNOWN.
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ 
+
    
+
 
    size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
     This function is deprecated, and is equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -1405,6 +1427,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
   For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
   but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
  @return : 0, or an error code (which can be tested using ZSTD_isError())
+  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  
 

    
 
@@ -1439,34 +1462,35 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
  *     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
  *
  * note: no dictionary will be used if dict == NULL or dictSize < 8
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
-    /**
- * This function is deprecated, and is equivalent to:
- *
- *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
- *     ZSTD_DCtx_refDDict(zds, ddict);
- *
- * note : ddict is referenced, it must outlive decompression session
- */
-size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
-/**
- * This function is deprecated, and is equivalent to:
- *
- *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
- *
- * re-use decompression parameters from previous init; saves dictionary loading
- */
-size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 
    
-
    Buffer-less and synchronous inner streaming functions
    +
    This function is deprecated, and is equivalent to:
    +     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+     ZSTD_DCtx_refDDict(zds, ddict);
+
+ note : ddict is referenced, it must outlive decompression session
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ 
+
    
+
+
    This function is deprecated, and is equivalent to:
    +     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+
+ re-use decompression parameters from previous init; saves dictionary loading
+ Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
+ 
+
    
+
+
    Buffer-less and synchronous inner streaming functions
       This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with several restrictions, documented below.
   Prefer normal streaming API for an easier experience.
  
 
    
 
-
    Buffer-less streaming compression (synchronous mode)
    +
    Buffer-less streaming compression (synchronous mode)
       A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -1502,7 +1526,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 

    
-
    Buffer-less streaming decompression (synchronous mode)
    +
    Buffer-less streaming decompression (synchronous mode)
       A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
@@ -1598,9 +1622,9 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
 
 
    typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 

    
-
    Block level API
    
+
    Block level API
    
 
-
        Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+
        Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
 
     A few rules to respect :
diff --git a/lib/zstd.h b/lib/zstd.h
index 22711d17b..b8669e661 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -197,9 +197,13 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 
 /*! ZSTD_compressCCtx() :
- *  Same as ZSTD_compress(), using an explicit ZSTD_CCtx
- *  The function will compress at requested compression level,
- *  ignoring any other parameter */
+ *  Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
+ *  Important : in order to behave similarly to `ZSTD_compress()`,
+ *  this function compresses at requested compression level,
+ *  __ignoring any other parameter__ .
+ *  If any advanced parameter was set using the advanced API,
+ *  they will all be reset. Only `compressionLevel` remains.
+ */
 ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
                                      void* dst, size_t dstCapacity,
                                const void* src, size_t srcSize,
@@ -234,7 +238,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
  *   using ZSTD_CCtx_set*() functions.
  *   Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
  *   "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
- *   They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()
+ *   __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
  *
  *   It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
  *
@@ -262,18 +266,26 @@ typedef enum {
 
     /* compression parameters
      * Note: When compressing with a ZSTD_CDict these parameters are superseded
-     * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
-     * for more info (superseded-by-cdict). */
-    ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
+     * by the parameters used to construct the ZSTD_CDict.
+     * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
+    ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
+                              * Note that exact compression parameters are dynamically determined,
+                              * depending on both compression level and srcSize (when known).
                               * Default level is ZSTD_CLEVEL_DEFAULT==3.
                               * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
                               * Note 1 : it's possible to pass a negative compression level.
-                              * Note 2 : setting a level sets all default values of other compression parameters */
+                              * Note 2 : setting a level resets all other compression parameters to default */
+    /* Advanced compression parameters :
+     * It's possible to pin down compression parameters to some specific values.
+     * In which case, these values are no longer dynamically selected by the compressor */
     ZSTD_c_windowLog=101,    /* Maximum allowed back-reference distance, expressed as power of 2.
+                              * This will set a memory budget for streaming decompression,
+                              * with larger values requiring more memory
+                              * and typically compressing more.
                               * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
                               * Special: value 0 means "use default windowLog".
                               * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
-                              *       requires explicitly allowing such window size at decompression stage if using streaming. */
+                              *       requires explicitly allowing such size at streaming decompression stage. */
     ZSTD_c_hashLog=102,      /* Size of the initial probe table, as a power of 2.
                               * Resulting memory usage is (1 << (hashLog+2)).
                               * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
@@ -284,13 +296,13 @@ typedef enum {
                               * Resulting memory usage is (1 << (chainLog+2)).
                               * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
                               * Larger tables result in better and slower compression.
-                              * This parameter is useless when using "fast" strategy.
+                              * This parameter is useless for "fast" strategy.
                               * It's still useful when using "dfast" strategy,
                               * in which case it defines a secondary probe table.
                               * Special: value 0 means "use default chainLog". */
     ZSTD_c_searchLog=104,    /* Number of search attempts, as a power of 2.
                               * More attempts result in better and slower compression.
-                              * This parameter is useless when using "fast" and "dFast" strategies.
+                              * This parameter is useless for "fast" and "dFast" strategies.
                               * Special: value 0 means "use default searchLog". */
     ZSTD_c_minMatch=105,     /* Minimum size of searched matches.
                               * Note that Zstandard can still find matches of smaller size,
@@ -345,7 +357,7 @@ typedef enum {
     ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
                               * Content size must be known at the beginning of compression.
                               * This is automatically the case when using ZSTD_compress2(),
-                              * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
+                              * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
     ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_c_dictIDFlag=202,   /* When applicable, dictionary's ID is written into frame header (default:1) */
 
@@ -364,7 +376,7 @@ typedef enum {
                               * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
                               * 0 means default, which is dynamically determined based on compression parameters.
                               * Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
-                              * The minimum size is automatically and transparently enforced */
+                              * The minimum size is automatically and transparently enforced. */
     ZSTD_c_overlapLog=402,   /* Control the overlap size, as a fraction of window size.
                               * The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
                               * It helps preserve compression ratio, while each job is compressed in parallel.
@@ -1128,15 +1140,6 @@ typedef enum {
 } ZSTD_dictLoadMethod_e;
 
 typedef enum {
-    /* Opened question : should we have a format ZSTD_f_auto ?
-     * Today, it would mean exactly the same as ZSTD_f_zstd1.
-     * But, in the future, should several formats become supported,
-     * on the compression side, it would mean "default format".
-     * On the decompression side, it would mean "automatic format detection",
-     * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
-     * Since meaning is a little different, another option could be to define different enums for compression and decompression.
-     * This question could be kept for later, when there are actually multiple formats to support,
-     * but there is also the question of pinning enum values, and pinning value `0` is especially important */
     ZSTD_f_zstd1 = 0,           /* zstd frame format, specified in zstd_compression_format.md (default) */
     ZSTD_f_zstd1_magicless = 1  /* Variant of zstd frame format, without initial 4-bytes magic number.
                                  * Useful to save 4 bytes per generated frame.
@@ -1375,7 +1378,8 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS
  *  Create a digested dictionary for compression
  *  Dictionary content is just referenced, not duplicated.
  *  As a consequence, `dictBuffer` **must** outlive CDict,
- *  and its content must remain unmodified throughout the lifetime of CDict. */
+ *  and its content must remain unmodified throughout the lifetime of CDict.
+ *  note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
 /*! ZSTD_getCParams() :
@@ -1402,7 +1406,9 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
 ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 
 /*! ZSTD_compress_advanced() :
- *  Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */
+ *  Note : this function is now DEPRECATED.
+ *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
+ *  This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */
 ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
                                           void* dst, size_t dstCapacity,
                                     const void* src, size_t srcSize,
@@ -1410,7 +1416,9 @@ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
                                           ZSTD_parameters params);
 
 /*! ZSTD_compress_usingCDict_advanced() :
- *  Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
+ *  Note : this function is now REDUNDANT.
+ *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
+ *  This prototype will be marked as deprecated and generate compilation warning in some future version */
 ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                                               void* dst, size_t dstCapacity,
                                         const void* src, size_t srcSize,
@@ -1660,8 +1668,13 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
  * pledgedSrcSize must be correct. If it is not known at init time, use
  * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
  * "0" also disables frame content size field. It may be enabled in the future.
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
-ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t
+ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
+                         int compressionLevel,
+                         unsigned long long pledgedSrcSize);
+
 /**! ZSTD_initCStream_usingDict() :
  * This function is deprecated, and is equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -1672,8 +1685,13 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
  * dict == NULL or dictSize < 8, in which case no dict is used.
  * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
  * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
-ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+ZSTDLIB_API size_t
+ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
+                     const void* dict, size_t dictSize,
+                           int compressionLevel);
+
 /**! ZSTD_initCStream_advanced() :
  * This function is deprecated, and is approximately equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -1684,21 +1702,29 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
  *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
  *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
  *
- * pledgedSrcSize must be correct. If srcSize is not known at init time, use
- * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
+ * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
+ * pledgedSrcSize must be correct.
+ * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
-ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t
+ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
+                    const void* dict, size_t dictSize,
+                          ZSTD_parameters params,
+                          unsigned long long pledgedSrcSize);
+
 /**! ZSTD_initCStream_usingCDict() :
  * This function is deprecated, and equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     ZSTD_CCtx_refCDict(zcs, cdict);
  *
  * note : cdict will just be referenced, and must outlive compression session
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
+
 /**! ZSTD_initCStream_usingCDict_advanced() :
- * This function is deprecated, and is approximately equivalent to:
+ *   This function is DEPRECATED, and is approximately equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
  *     for ((fParam, value) : fParams) {
@@ -1710,8 +1736,13 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDi
  * same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
  * pledgedSrcSize must be correct. If srcSize is not known at init time, use
  * value ZSTD_CONTENTSIZE_UNKNOWN.
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
-ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t
+ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+                               const ZSTD_CDict* cdict,
+                                     ZSTD_frameParameters fParams,
+                                     unsigned long long pledgedSrcSize);
 
 /*! ZSTD_resetCStream() :
  * This function is deprecated, and is equivalent to:
@@ -1726,6 +1757,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const
  *  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
  *  but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
  * @return : 0, or an error code (which can be tested using ZSTD_isError())
+ *  Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
 ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
@@ -1771,8 +1803,10 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
  *     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
  *
  * note: no dictionary will be used if dict == NULL or dictSize < 8
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
 ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+
 /**
  * This function is deprecated, and is equivalent to:
  *
@@ -1780,14 +1814,17 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic
  *     ZSTD_DCtx_refDDict(zds, ddict);
  *
  * note : ddict is referenced, it must outlive decompression session
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
 ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+
 /**
  * This function is deprecated, and is equivalent to:
  *
  *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
  *
  * re-use decompression parameters from previous init; saves dictionary loading
+ * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
 ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 
@@ -1961,7 +1998,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
 
 /*!
     Block functions produce and decode raw zstd blocks, without frame metadata.
-    Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+    Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
 
     A few rules to respect :

From 5d5c895b18ea54c577fdc4d89c8fd334eba4e19a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 22 Oct 2019 14:57:15 -0700
Subject: [PATCH 105/106] fix initCStream_advanced() for fast strategies

Compression ratio of fast strategies (levels 1 & 2)
was seriously reduced, due to accidental disabling of Literals compression.

Credit to @QrczakMK, which perfectly described the issue, and implementation details,
making the fix straightforward.

Example : initCStream with level 1 on synthetic sample P50 :
Before : 5,273,976 bytes
After  : 3,154,678 bytes
ZSTD_compress (for comparison) : 3,154,550

Fix #1787.

To follow : refactor the test which was supposed to catch this issue (and failed)
---
 lib/compress/zstd_compress.c | 32 ++++++++++++++++----------------
 tests/fullbench.c            |  3 +--
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 6113df4b2..ded25d8da 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -238,10 +238,10 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete
     RETURN_ERROR_IF(!cctxParams, GENERIC);
     FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) );
     memset(cctxParams, 0, sizeof(*cctxParams));
+    assert(!ZSTD_checkCParams(params.cParams));
     cctxParams->cParams = params.cParams;
     cctxParams->fParams = params.fParams;
     cctxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* should not matter, as all cParams are presumed properly defined */
-    assert(!ZSTD_checkCParams(params.cParams));
     return 0;
 }
 
@@ -251,10 +251,10 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams(
         const ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
 {
     ZSTD_CCtx_params ret = *cctxParams;
+    assert(!ZSTD_checkCParams(params.cParams));
     ret.cParams = params.cParams;
     ret.fParams = params.fParams;
     ret.compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* should not matter, as all cParams are presumed properly defined */
-    assert(!ZSTD_checkCParams(params.cParams));
     return ret;
 }
 
@@ -533,33 +533,33 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         if (value) {  /* 0 : does not change current level */
             CCtxParams->compressionLevel = value;
         }
-        if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel;
+        if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel;
         return 0;  /* return type (size_t) cannot represent negative values */
     }
 
     case ZSTD_c_windowLog :
         if (value!=0)   /* 0 => use default */
             BOUNDCHECK(ZSTD_c_windowLog, value);
-        CCtxParams->cParams.windowLog = value;
+        CCtxParams->cParams.windowLog = (U32)value;
         return CCtxParams->cParams.windowLog;
 
     case ZSTD_c_hashLog :
         if (value!=0)   /* 0 => use default */
             BOUNDCHECK(ZSTD_c_hashLog, value);
-        CCtxParams->cParams.hashLog = value;
+        CCtxParams->cParams.hashLog = (U32)value;
         return CCtxParams->cParams.hashLog;
 
     case ZSTD_c_chainLog :
         if (value!=0)   /* 0 => use default */
             BOUNDCHECK(ZSTD_c_chainLog, value);
-        CCtxParams->cParams.chainLog = value;
+        CCtxParams->cParams.chainLog = (U32)value;
         return CCtxParams->cParams.chainLog;
 
     case ZSTD_c_searchLog :
         if (value!=0)   /* 0 => use default */
             BOUNDCHECK(ZSTD_c_searchLog, value);
-        CCtxParams->cParams.searchLog = value;
-        return value;
+        CCtxParams->cParams.searchLog = (U32)value;
+        return (size_t)value;
 
     case ZSTD_c_minMatch :
         if (value!=0)   /* 0 => use default */
@@ -3810,7 +3810,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
             if (cctx->mtctx == NULL) {
                 DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
                             params.nbWorkers);
-                cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem);
+                cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem);
                 RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation);
             }
             /* mt compression */
@@ -3938,8 +3938,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
     { 19, 12, 13,  1,  6,  1, ZSTD_fast    },  /* base for negative levels */
     { 19, 13, 14,  1,  7,  0, ZSTD_fast    },  /* level  1 */
     { 20, 15, 16,  1,  6,  0, ZSTD_fast    },  /* level  2 */
-    { 21, 16, 17,  1,  5,  1, ZSTD_dfast   },  /* level  3 */
-    { 21, 18, 18,  1,  5,  1, ZSTD_dfast   },  /* level  4 */
+    { 21, 16, 17,  1,  5,  0, ZSTD_dfast   },  /* level  3 */
+    { 21, 18, 18,  1,  5,  0, ZSTD_dfast   },  /* level  4 */
     { 21, 18, 19,  2,  5,  2, ZSTD_greedy  },  /* level  5 */
     { 21, 19, 19,  3,  5,  4, ZSTD_greedy  },  /* level  6 */
     { 21, 19, 19,  3,  5,  8, ZSTD_lazy    },  /* level  7 */
@@ -3963,8 +3963,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
     /* W,  C,  H,  S,  L,  T, strat */
     { 18, 12, 13,  1,  5,  1, ZSTD_fast    },  /* base for negative levels */
     { 18, 13, 14,  1,  6,  0, ZSTD_fast    },  /* level  1 */
-    { 18, 14, 14,  1,  5,  1, ZSTD_dfast   },  /* level  2 */
-    { 18, 16, 16,  1,  4,  1, ZSTD_dfast   },  /* level  3 */
+    { 18, 14, 14,  1,  5,  0, ZSTD_dfast   },  /* level  2 */
+    { 18, 16, 16,  1,  4,  0, ZSTD_dfast   },  /* level  3 */
     { 18, 16, 17,  2,  5,  2, ZSTD_greedy  },  /* level  4.*/
     { 18, 18, 18,  3,  5,  2, ZSTD_greedy  },  /* level  5.*/
     { 18, 18, 19,  3,  5,  4, ZSTD_lazy    },  /* level  6.*/
@@ -3990,8 +3990,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
     { 17, 12, 12,  1,  5,  1, ZSTD_fast    },  /* base for negative levels */
     { 17, 12, 13,  1,  6,  0, ZSTD_fast    },  /* level  1 */
     { 17, 13, 15,  1,  5,  0, ZSTD_fast    },  /* level  2 */
-    { 17, 15, 16,  2,  5,  1, ZSTD_dfast   },  /* level  3 */
-    { 17, 17, 17,  2,  4,  1, ZSTD_dfast   },  /* level  4 */
+    { 17, 15, 16,  2,  5,  0, ZSTD_dfast   },  /* level  3 */
+    { 17, 17, 17,  2,  4,  0, ZSTD_dfast   },  /* level  4 */
     { 17, 16, 17,  3,  4,  2, ZSTD_greedy  },  /* level  5 */
     { 17, 17, 17,  3,  4,  4, ZSTD_lazy    },  /* level  6 */
     { 17, 17, 17,  3,  4,  8, ZSTD_lazy2   },  /* level  7 */
@@ -4016,7 +4016,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
     { 14, 12, 13,  1,  5,  1, ZSTD_fast    },  /* base for negative levels */
     { 14, 14, 15,  1,  5,  0, ZSTD_fast    },  /* level  1 */
     { 14, 14, 15,  1,  4,  0, ZSTD_fast    },  /* level  2 */
-    { 14, 14, 15,  2,  4,  1, ZSTD_dfast   },  /* level  3 */
+    { 14, 14, 15,  2,  4,  0, ZSTD_dfast   },  /* level  3 */
     { 14, 14, 14,  4,  4,  2, ZSTD_greedy  },  /* level  4 */
     { 14, 14, 14,  3,  4,  4, ZSTD_lazy    },  /* level  5.*/
     { 14, 14, 14,  4,  4,  8, ZSTD_lazy2   },  /* level  6 */
diff --git a/tests/fullbench.c b/tests/fullbench.c
index 0e2761e11..be49b1428 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -45,7 +45,6 @@
 #define NBLOOPS    6
 #define TIMELOOP_S 2
 
-#define KNUTH      2654435761U
 #define MAX_MEM    (1984 MB)
 
 #define DEFAULT_CLEVEL 1
@@ -722,7 +721,7 @@ static int usage_advanced(const char* exename)
     DISPLAY( "\nAdvanced options :\n");
     DISPLAY( " -b#    : test only function # \n");
     DISPLAY( " -l#    : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL);
-    DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n");
+    DISPLAY( "--zstd= : custom parameter selection. Format same as zstdcli \n");
     DISPLAY( " -P#    : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
     DISPLAY( " -B#    : sample size (default : %u)\n", (unsigned)kSampleSizeDefault);
     DISPLAY( " -i#    : iteration loops [1-9](default : %i)\n", NBLOOPS);

From 8adecc73b0b9c01450c842b222491cfeacf5e308 Mon Sep 17 00:00:00 2001
From: Bimba Shrestha 
Date: Fri, 25 Oct 2019 12:04:54 -0700
Subject: [PATCH 106/106] Running playtests.sh on PRs too

---
 appveyor.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 87fa5c12d..3a4803363 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -264,5 +264,6 @@
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
   - if [%HOST%]==[mingw] (
       set "CC=%COMPILER%" &&
-      make check
+      make check &&
+      sh -e playTests.sh --test-large-data
     )