From 33eb7ac6b6e152ce9263fdb1f84eb9bf50abe1eb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 12 May 2017 12:36:11 -0700 Subject: [PATCH 001/109] updated Advanced API proposal only declarations in zstd.h --- doc/zstd_manual.html | 159 ++++++++++++++++++++++++++++++++++ lib/zstd.h | 198 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 357 insertions(+) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 59d146874..9b0973164 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -471,6 +471,165 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);

Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters


+

New experimental advanced parameters API



+
typedef enum {
+    /* compression parameters */
+    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table (default:3) */
+    ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
+                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+                              * default value : set through compressionLevel */
+    ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
+                              * Resulting table size is (1 << (hashLog+2)).
+                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+                              * Larger tables improve compression ratio of strategies <= dFast,
+                              * and improve speed of strategies > dFast */
+    ZSTD_p_chainLog,         /* Size of the full-search table, as a power of 2.
+                              * Resulting table size is (1 << (chainLog+2)).
+                              * Larger tables result in better and slower compression.
+                              * This parameter is useless when using "fast" strategy */
+    ZSTD_p_searchLog,        /* Number of search attempts, as a power of 2.
+                              * More attempts result in better and slower compression.
+                              * This parameter is useless when using "fast" and "dFast" strategies */
+    ZSTD_p_minMatchLength,   /* Minimum match size (except for repeat-matches, which limit is hard-coded).
+                              * Larger values make compression and decompression faster, but decrease compression ratio
+                              * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
+                              * Note that currently, for all strategies < btopt, effective minimum is 4.
+                              * Note that currently, for all strategies > fast, effective maximum is 6. */
+    ZSTD_p_targetLength,     /* Only useful for strategies >= btopt.
+                              * Length of Match considered "good enough" to stop search.
+                              * Larger values make compression stronger and slower. */
+    ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
+                              * Cast selected strategy into unsigned for ZSTD_CCtx_setParameter() compatibility.
+                              * The higher the value of selected strategy, the more complex it is,
+                              * resulting in stronger and slower compression */
+#if 0
+    ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
+                              * Can be set to a more precise value than windowLog.
+                              * Will be transparently reduced to closest possible inferior value
+                              * (see Zstandard compression format) */
+                             /* Not ready yet ! */
+#endif
+
+    /* frame parameters */
+    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
+    ZSTD_p_contentChecksumFlag, /* A 32-bits content checksum is calculated and written at end of frame (default:0) */
+    ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
+
+    /* dictionary parameters */
+    ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
+                              * This avoids duplicating dictionary content.
+                              * But it also requires that dictionary buffer outlives its user (CCtx or CDict) */
+                             /* Not ready yet ! */
+    ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
+                             /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
+
+#if 0
+    /* multi-threading parameters (not ready yet !) */
+    ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
+                              * More threads improve speed, but increases also memory usage */
+    ZSTDMT_p_jobSize,        /* Size of a compression job. Each job is compressed in parallel.
+                              * 0 means default, which is dynamically determined based on compression parameters.
+                              * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
+                              * The minimum size is automatically and transparently enforced */
+    ZSTDMT_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
+                              * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
+#endif
+
+    /* advanced parameters - may not remain available after API update */
+    ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
+                              * even when referencing into Dictionary content
+                              * default : 0 when using a CDict, 1 when using a Prefix */
+} ZSTD_cParameter;
+


+
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
+
  Set one compression parameter, selected by enum ZSTD_cParameter.
+  @result : 0, or an error code (which can be tested with ZSTD_isError()) 
+


+
+
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
  Total input data size to be compressed into a single frame.
+  This value will be controlled at the end, and result in error if not respected.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Note 1 : 0 means zero, empty.
+           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for all new compression jobs.
+  Note 2 : If all data is provided and consumed in a single round,
+           this value is overriden by srcSize instead. 
+


+
+
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
+
  Reference a prefix (content-only dictionary) to bootstrap next compression job.
+  Decompression will have to use same prefix.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
+  Note 1 : Prefix content is referenced. It must outlive compression job.
+  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+           For this reason, compression parameters cannot be changed anymore after loading a prefix.
+           It's also a CPU-heavy operation, with non-negligible impact on latency.
+  Note 3 : Prefix is only used once. Tables are discarded at end of compression job.
+           If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict 
+


+
+
ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
+size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
+size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
+
  Create a CDict object which is still mutable after creation.
+  It allows usage of ZSTD_CDict_setParameter().
+  Once all compression parameters are selected,
+  it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
+  Dictionary content will be copied internally, except if ZSTD_p_refDictContent is used.
+  After loading the dictionary, no more change is possible.
+  The only remaining operation is to free CDict object.
+  Note : An unfinished CDict behaves the same as a NULL CDict when referenced into a CCtx.
+ 
+


+
+
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);  /* Not ready yet ! */
+
  Add a prepared dictionary to cctx, it will used for next compression jobs.
+  Note that compression parameters will be enforced from within CDict.
+  Currently, they supercede any compression parameter previously set within CCtx.
+  The dictionary will remain valid for all future compression jobs performed using the same cctx.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Special : adding a NULL CDict means "return to no-dictionary mode".
+  Note 1 : Currently, only one dictionary can be managed.
+           Adding a new dictionary effectively "discards" any previous one.
+  Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
+ 
+


+
+
typedef enum {
+    ZSTD_e_continue,  /* continue sending data, encoder transparently decides when to output result, depending on optimal conditions */
+    ZSTD_e_flush,     /* flush any data provided and buffered so far - frame will continue, future data can still reference previous data for better compression */
+    ZSTD_e_end        /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
+} ZSTD_EndDirective;
+


+
size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
+                              void* dst, size_t dstCapacity, size_t* dstPos,
+                        const void* src, size_t srcSize, size_t* srcPos,
+                              ZSTD_EndDirective endOp);
+
  Behave about the same as ZSTD_compressStream. To note :
+  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_setCCtxParameter()
+  - Compression parameters cannot be changed once compression is started.
+  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
+  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
+  - @return provides the amount of data ready to flush and still within internal buffers
+            or an error code, which can be tested using ZSTD_isError().
+            if @return != 0, flush is not fully completed, so it must be called again to empty internal buffers.
+  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
+            only ZSTD_e_end and ZSTD_e_flush operations are allowed.
+            It is necessary to fully flush internal buffers
+            before changing compression parameters or start a new compression job.
+ 
+


+
+
size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
+
  Return a CCtx to clean state.
+  Useful after an error, or to interrupt an ongoing compression job and start a new one.
+  It's allowed to change compression parameters after a reset.
+  Any internal data not yet flushed is cancelled.
+ 
+


+
 
Advanced decompression functions

 
 
unsigned ZSTD_isFrame(const void* buffer, size_t size);
diff --git a/lib/zstd.h b/lib/zstd.h
index 256117662..be2168bf9 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -501,6 +501,8 @@ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
  *  Create a ZSTD compression context using external alloc and free functions */
 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 
+
+/* !!! Soon to be deprecated !!! */
 typedef enum {
     ZSTD_p_forceWindow,   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
     ZSTD_p_forceRawDict   /* Force loading dictionary in "content-only" mode (no header analysis) */
@@ -556,6 +558,202 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                             const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
 
 
+/*===   New experimental advanced parameters API   ===*/
+
+/* notes on API design :
+ *   In below proposal, parameters are pushed one by one into an existing CCtx,
+ *   and then applied on all following compression jobs.
+ *   When no parameter is ever provided, CCtx is created with compression level 3.
+ *
+ *   Another approach could be to load parameters into an intermediate _opaque_ object.
+ *   The object would then be loaded into CCtx (like ZSTD_compress_advanced())
+ *   This approach would be compatible with ZSTD_createCDict_advanced().
+ *   But it's a bit more cumbersome for CCtx, because it requires to manage an additional object.
+ *
+ *   Below proposal push parameters directly into CCtx.
+ *   It's a bit more complex for CDict, as advanced version now requires 3 steps.
+ *   (basic CDict API remains the same).
+ *   See API details below
+ */
+
+/* notes on naming convention :
+ *   Initially, the API favored naming like ZSTD_setCCtxParameter() .
+ *   In this proposal, this is changed into ZSTD_CCtx_setParameter() .
+ *   The main idea is that it identifies more clearly the target object type.
+ *   It feels clearer when considering other potential variants :
+ *   ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
+ *   ZSTD_DCtx_setParameter()  (rather than ZSTD_setDCtxParameter() )
+ *   The first verion looks clearer.
+ */
+
+typedef enum {
+    /* compression parameters */
+    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table (default:3) */
+    ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
+                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+                              * default value : set through compressionLevel */
+    ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
+                              * Resulting table size is (1 << (hashLog+2)).
+                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+                              * Larger tables improve compression ratio of strategies <= dFast,
+                              * and improve speed of strategies > dFast */
+    ZSTD_p_chainLog,         /* Size of the full-search table, as a power of 2.
+                              * Resulting table size is (1 << (chainLog+2)).
+                              * Larger tables result in better and slower compression.
+                              * This parameter is useless when using "fast" strategy */
+    ZSTD_p_searchLog,        /* Number of search attempts, as a power of 2.
+                              * More attempts result in better and slower compression.
+                              * This parameter is useless when using "fast" and "dFast" strategies */
+    ZSTD_p_minMatchLength,   /* Minimum match size (except for repeat-matches, which limit is hard-coded).
+                              * Larger values make compression and decompression faster, but decrease compression ratio
+                              * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
+                              * Note that currently, for all strategies < btopt, effective minimum is 4.
+                              * Note that currently, for all strategies > fast, effective maximum is 6. */
+    ZSTD_p_targetLength,     /* Only useful for strategies >= btopt.
+                              * Length of Match considered "good enough" to stop search.
+                              * Larger values make compression stronger and slower. */
+    ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
+                              * Cast selected strategy into unsigned for ZSTD_CCtx_setParameter() compatibility.
+                              * The higher the value of selected strategy, the more complex it is,
+                              * resulting in stronger and slower compression */
+#if 0
+    ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
+                              * Can be set to a more precise value than windowLog.
+                              * Will be transparently reduced to closest possible inferior value
+                              * (see Zstandard compression format) */
+                             /* Not ready yet ! */
+#endif
+
+    /* frame parameters */
+    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
+    ZSTD_p_contentChecksumFlag, /* A 32-bits content checksum is calculated and written at end of frame (default:0) */
+    ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
+
+    /* dictionary parameters */
+    ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
+                              * This avoids duplicating dictionary content.
+                              * But it also requires that dictionary buffer outlives its user (CCtx or CDict) */
+                             /* Not ready yet ! */
+    ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
+                             /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
+
+#if 0
+    /* multi-threading parameters (not ready yet !) */
+    ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
+                              * More threads improve speed, but increases also memory usage */
+    ZSTDMT_p_jobSize,        /* Size of a compression job. Each job is compressed in parallel.
+                              * 0 means default, which is dynamically determined based on compression parameters.
+                              * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
+                              * The minimum size is automatically and transparently enforced */
+    ZSTDMT_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
+                              * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
+#endif
+
+    /* advanced parameters - may not remain available after API update */
+    ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
+                              * even when referencing into Dictionary content
+                              * default : 0 when using a CDict, 1 when using a Prefix */
+} ZSTD_cParameter;
+
+
+/*! ZSTD_CCtx_setParameter() :
+ *  Set one compression parameter, selected by enum ZSTD_cParameter.
+ *  @result : 0, or an error code (which can be tested with ZSTD_isError()) */
+ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
+
+/*! ZSTD_CCtx_setPledgedSrcSize() :
+ *  Total input data size to be compressed into a single frame.
+ *  This value will be controlled at the end, and result in error if not respected.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : 0 means zero, empty.
+ *           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+ *           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for all new compression jobs.
+ *  Note 2 : If all data is provided and consumed in a single round,
+ *           this value is overriden by srcSize instead. */
+ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
+/*! ZSTD_CCtx_refPrefix() :
+ *  Reference a prefix (content-only dictionary) to bootstrap next compression job.
+ *  Decompression will have to use same prefix.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
+ *  Note 1 : Prefix content is referenced. It must outlive compression job.
+ *  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ *           For this reason, compression parameters cannot be changed anymore after loading a prefix.
+ *           It's also a CPU-heavy operation, with non-negligible impact on latency.
+ *  Note 3 : Prefix is only used once. Tables are discarded at end of compression job.
+ *           If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
+
+
+/*! ZSTD_CDict_createEmpty() :
+ *  Create a CDict object which is still mutable after creation.
+ *  It allows usage of ZSTD_CDict_setParameter().
+ *  Once all compression parameters are selected,
+ *  it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
+ *  Dictionary content will be copied internally, except if ZSTD_p_refDictContent is used.
+ *  After loading the dictionary, no more change is possible.
+ *  The only remaining operation is to free CDict object.
+ *  Note : An unfinished CDict behaves the same as a NULL CDict when referenced into a CCtx.
+ */
+ZSTDLIB_API ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
+ZSTDLIB_API size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
+ZSTDLIB_API size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
+
+/*! ZSTD_CCtx_refCDict() :
+ *  Add a prepared dictionary to cctx, it will used for next compression jobs.
+ *  Note that compression parameters will be enforced from within CDict.
+ *  Currently, they supercede any compression parameter previously set within CCtx.
+ *  The dictionary will remain valid for all future compression jobs performed using the same cctx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : adding a NULL CDict means "return to no-dictionary mode".
+ *  Note 1 : Currently, only one dictionary can be managed.
+ *           Adding a new dictionary effectively "discards" any previous one.
+ *  Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);  /* Not ready yet ! */
+
+
+#if 0
+// Target advanced compression API
+// Not ready yet !!!
+
+typedef enum {
+    ZSTD_e_continue,  /* continue sending data, encoder transparently decides when to output result, depending on optimal conditions */
+    ZSTD_e_flush,     /* flush any data provided and buffered so far - frame will continue, future data can still reference previous data for better compression */
+    ZSTD_e_end        /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
+} ZSTD_EndDirective;
+
+/*! ZSTD_compressStream_generic() :
+ *  Behave about the same as ZSTD_compressStream. To note :
+ *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_setCCtxParameter()
+ *  - Compression parameters cannot be changed once compression is started.
+ *  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
+ *  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
+ *  - @return provides the amount of data ready to flush and still within internal buffers
+ *            or an error code, which can be tested using ZSTD_isError().
+ *            if @return != 0, flush is not fully completed, so it must be called again to empty internal buffers.
+ *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
+ *            only ZSTD_e_end and ZSTD_e_flush operations are allowed.
+ *            It is necessary to fully flush internal buffers
+ *            before changing compression parameters or start a new compression job.
+ */
+ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
+                                          void* dst, size_t dstCapacity, size_t* dstPos,
+                                    const void* src, size_t srcSize, size_t* srcPos,
+                                          ZSTD_EndDirective endOp);
+
+/*! ZSTD_CCtx_reset() :
+ *  Return a CCtx to clean state.
+ *  Useful after an error, or to interrupt an ongoing compression job and start a new one.
+ *  It's allowed to change compression parameters after a reset.
+ *  Any internal data not yet flushed is cancelled.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
+
+#endif
+
+
 /*--- Advanced decompression functions ---*/
 
 /*! ZSTD_isFrame() :

From db8e21d5a05b6370b01dc76b2fef4377825189b5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 12 May 2017 13:46:49 -0700
Subject: [PATCH 002/109] made ZSTD_compress_generic() definition accessible

note that the implementation is not done yet.
---
 doc/zstd_manual.html         | 6 +++---
 lib/compress/zstd_compress.c | 6 +++---
 lib/zstd.h                   | 9 ++++-----
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 9b0973164..a06d4b037 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -574,13 +574,13 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
 size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
 size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
 
  Create a CDict object which is still mutable after creation.
-  It allows usage of ZSTD_CDict_setParameter().
+  It's the only one case allowing usage of ZSTD_CDict_setParameter().
   Once all compression parameters are selected,
   it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
-  Dictionary content will be copied internally, except if ZSTD_p_refDictContent is used.
+  Dictionary content will be copied internally (except if ZSTD_p_refDictContent is set).
   After loading the dictionary, no more change is possible.
   The only remaining operation is to free CDict object.
-  Note : An unfinished CDict behaves the same as a NULL CDict when referenced into a CCtx.
+  Note : An unfinished CDict behaves the same as a NULL CDict if referenced into a CCtx.
  
 


 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index edf4609f7..0db9d1cd0 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2468,14 +2468,14 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
 }
 
 
-/*! ZSTD_compress_generic() :
+/*! ZSTD_compress_frameChunk() :
 *   Compress a chunk of data into one or multiple blocks.
 *   All blocks will be terminated, all input will be consumed.
 *   Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
 *   Frame is supposed already started (header already produced)
 *   @return : compressed size, or an error code
 */
-static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
+static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
                                      void* dst, size_t dstCapacity,
                                const void* src, size_t srcSize,
                                      U32 lastFrameChunk)
@@ -2630,7 +2630,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
 
     if (srcSize) {
         size_t const cSize = frame ?
-                             ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
+                             ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
                              ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
         if (ZSTD_isError(cSize)) return cSize;
         cctx->consumedSrcSize += srcSize;
diff --git a/lib/zstd.h b/lib/zstd.h
index be2168bf9..fcefc7af6 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -688,13 +688,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size
 
 /*! ZSTD_CDict_createEmpty() :
  *  Create a CDict object which is still mutable after creation.
- *  It allows usage of ZSTD_CDict_setParameter().
+ *  It's the only one case allowing usage of ZSTD_CDict_setParameter().
  *  Once all compression parameters are selected,
  *  it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
- *  Dictionary content will be copied internally, except if ZSTD_p_refDictContent is used.
+ *  Dictionary content will be copied internally (except if ZSTD_p_refDictContent is set).
  *  After loading the dictionary, no more change is possible.
  *  The only remaining operation is to free CDict object.
- *  Note : An unfinished CDict behaves the same as a NULL CDict when referenced into a CCtx.
+ *  Note : An unfinished CDict behaves the same as a NULL CDict if referenced into a CCtx.
  */
 ZSTDLIB_API ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
 ZSTDLIB_API size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
@@ -714,7 +714,6 @@ ZSTDLIB_API size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict
 ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);  /* Not ready yet ! */
 
 
-#if 0
 // Target advanced compression API
 // Not ready yet !!!
 
@@ -749,9 +748,9 @@ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
  *  It's allowed to change compression parameters after a reset.
  *  Any internal data not yet flushed is cancelled.
  */
+// Not ready yet !!!
 ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
 
-#endif
 
 
 /*--- Advanced decompression functions ---*/

From ef738c1b238b81c705b4bf511cad7821d771a189 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 12 May 2017 13:53:46 -0700
Subject: [PATCH 003/109] better error code when compressing using NULL CDict

which is not allowed (but detected, and generates an error).
---
 lib/compress/zstd_compress.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 0db9d1cd0..57d788e59 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -177,6 +177,7 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
            + cctx->outBuffSize + cctx->inBuffSize;
 }
 
+/* older variant; will be deprecated */
 size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
 {
     switch(param)
@@ -3078,7 +3079,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
     ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
 {
-    if (cdict==NULL) return ERROR(GENERIC);  /* does not support NULL cdict */
+    if (cdict==NULL) return ERROR(dictionary_wrong);  /* does not support NULL cdict */
     DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced : dictIDFlag == %u \n", !fParams.noDictIDFlag);
     if (cdict->dictContentSize)
         CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) )
@@ -3231,7 +3232,7 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
  * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
 size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams)
 {
-    if (!cdict) return ERROR(GENERIC);   /* cannot handle NULL cdict (does not know what to do) */
+    if (!cdict) return ERROR(dictionary_wrong);  /* cannot handle NULL cdict (does not know what to do) */
     {   ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
         params.fParams = fParams;
         zcs->cdict = cdict;
@@ -3242,7 +3243,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
 /* note : cdict must outlive compression session */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 {
-    ZSTD_frameParameters const fParams = { 0 /* content */, 0 /* checksum */, 0 /* noDictID */ };
+    ZSTD_frameParameters const fParams = { 0 /* contentSize */, 0 /* checksum */, 0 /* hideDictID */ };
     return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, 0, fParams);
 }
 

From b0edb7fb0eb90dfe51682d806f8282b6f1ff6b3a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 12 May 2017 15:31:53 -0700
Subject: [PATCH 004/109] added ZSTD_CCtx_setParameter()

---
 doc/zstd_manual.html         |  11 +--
 lib/compress/zstd_compress.c | 132 +++++++++++++++++++++++++++++++++--
 lib/zstd.h                   |   8 ++-
 3 files changed, 138 insertions(+), 13 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index a06d4b037..68efb6cc8 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -474,7 +474,9 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
 
New experimental advanced parameters API


 
typedef enum {
     /* compression parameters */
-    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table (default:3) */
+    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
+                              * Default level is 3.
+                              * Special: value 0 means "do not change cLevel". */
     ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
                               * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
                               * default value : set through compressionLevel */
@@ -491,7 +493,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
                               * More attempts result in better and slower compression.
                               * This parameter is useless when using "fast" and "dFast" strategies */
     ZSTD_p_minMatchLength,   /* Minimum match size (except for repeat-matches, which limit is hard-coded).
-                              * Larger values make compression and decompression faster, but decrease compression ratio
+                              * Larger values make faster compression and decompression, but decrease ratio.
                               * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
                               * Note that currently, for all strategies < btopt, effective minimum is 4.
                               * Note that currently, for all strategies > fast, effective maximum is 6. */
@@ -499,7 +501,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
                               * Length of Match considered "good enough" to stop search.
                               * Larger values make compression stronger and slower. */
     ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
-                              * Cast selected strategy into unsigned for ZSTD_CCtx_setParameter() compatibility.
+                              * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
                               * The higher the value of selected strategy, the more complex it is,
                               * resulting in stronger and slower compression */
 #if 0
@@ -622,7 +624,8 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
  
 


 
-
size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
+
// Not ready yet !!!
+size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
 
  Return a CCtx to clean state.
   Useful after an error, or to interrupt an ongoing compression job and start a new one.
   It's allowed to change compression parameters after a reset.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 57d788e59..e17521565 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -8,6 +8,13 @@
  */
 
 
+/*-*************************************
+*  Tuning parameters
+***************************************/
+#ifndef ZSTD_DEFAULT_CLEVEL
+#  define ZSTD_DEFAULT_CLEVEL 3
+#endif
+
 /*-*************************************
 *  Dependencies
 ***************************************/
@@ -97,6 +104,7 @@ struct ZSTD_CCtx_s {
     U32   rep[ZSTD_REP_NUM];
     U32   repToConfirm[ZSTD_REP_NUM];
     U32   dictID;
+    int   compressionLevel;
     ZSTD_parameters params;
     void* workSpace;
     size_t workSpaceSize;
@@ -151,6 +159,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
     if (!cctx) return NULL;
     memset(cctx, 0, sizeof(ZSTD_CCtx));
     cctx->customMem = customMem;
+    cctx->compressionLevel = ZSTD_DEFAULT_CLEVEL;
     return cctx;
 }
 
@@ -177,6 +186,11 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
            + cctx->outBuffSize + cctx->inBuffSize;
 }
 
+/* private API call, for dictBuilder only */
+const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
+
+static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx) { return cctx->params; }
+
 /* older variant; will be deprecated */
 size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
 {
@@ -188,14 +202,121 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
     }
 }
 
-const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx)   /* hidden interface */
+size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
 {
-    return &(ctx->seqStore);
+#   define CLAMPCHECK(val,min,max) { if ((valmax)) return ERROR(compressionParameter_unsupported); }
+#   define LOADCPARAMS(cParams) \
+            if (cctx->compressionLevel!=0)  {               \
+                cParams = ZSTD_getCParams(                  \
+                    cctx->compressionLevel,                 \
+                    cctx->frameContentSize, 0);   /* dictSize unknown at this stage */ \
+                cctx->compressionLevel = 0;                 \
+            }
+
+
+    switch(param)
+    {
+    case ZSTD_p_compressionLevel :
+            if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel();   /* cap max compression level */
+            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+            cctx->compressionLevel = value;
+            return 0;
+
+    case ZSTD_p_windowLog :
+            CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+            LOADCPARAMS(cctx->params.cParams);
+            cctx->params.cParams.windowLog = value;
+            return 0;
+
+    case ZSTD_p_hashLog :
+            CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
+            LOADCPARAMS(cctx->params.cParams);
+            cctx->params.cParams.hashLog = value;
+            return 0;
+
+    case ZSTD_p_chainLog :
+            CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
+            LOADCPARAMS(cctx->params.cParams);
+            cctx->params.cParams.chainLog = value;
+            return 0;
+
+    case ZSTD_p_searchLog :
+            CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
+            LOADCPARAMS(cctx->params.cParams);
+            cctx->params.cParams.searchLog = value;
+            return 0;
+
+    case ZSTD_p_minMatchLength :
+            CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+            LOADCPARAMS(cctx->params.cParams);
+            cctx->params.cParams.searchLength = value;
+            return 0;
+
+    case ZSTD_p_targetLength :
+            CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
+            LOADCPARAMS(cctx->params.cParams);
+            cctx->params.cParams.targetLength = value;
+            return 0;
+
+    case ZSTD_p_compressionStrategy :
+            if (value > (unsigned)ZSTD_btultra) return ERROR(compressionParameter_unsupported);
+            LOADCPARAMS(cctx->params.cParams);
+            cctx->params.cParams.strategy = (ZSTD_strategy)value;
+            return 0;
+
+#if 0
+    case ZSTD_p_windowSize :   /* to be done later */
+            return ERROR(compressionParameter_unsupported);
+#endif
+
+    case ZSTD_p_contentSizeFlag : /* Content size will be written in frame header _when known_ (default:1) */
+            cctx->params.fParams.contentSizeFlag = value>0;
+            return 0;
+
+    case ZSTD_p_contentChecksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
+            cctx->params.fParams.checksumFlag = value>0;
+            return 0;
+
+    case ZSTD_p_dictIDFlag : /* When applicable, the dictID of used dictionary will be provided in frame header (default:1) */
+            cctx->params.fParams.noDictIDFlag = value==0;
+            return 0;
+
+    case ZSTD_p_refDictContent :   /* to be done later */
+            return ERROR(compressionParameter_unsupported);
+
+    case ZSTD_p_forceMaxWindow :  /* Force back-references to remain < windowSize,
+                                   * even when referencing into Dictionary content
+                                   * default : 0 when using a CDict, 1 when using a Prefix */
+            cctx->forceWindow = value>0;
+            cctx->loadedDictEnd = 0;
+            return 0;
+
+    case ZSTD_p_rawContentDict :  /* load dictionary in "content-only" mode (no header analysis) (default:0) */
+            cctx->forceRawDict = value>0;
+            return 0;
+
+    default: return ERROR(parameter_unknown);
+    }
 }
 
-static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx)
+ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
-    return cctx->params;
+    cctx->frameContentSize = pledgedSrcSize;
+    return 0;
+}
+
+/* Not ready yet ! */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
+{
+    (void)cctx; (void)prefix; (void)prefixSize; /* to be done later */
+    return ERROR(compressionParameter_unsupported);
+}
+
+/* Not ready yet ! */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
+{
+    (void)cctx; (void)cdict;  /* to be done later */
+    return ERROR(compressionParameter_unsupported);
 }
 
 
@@ -3472,7 +3593,6 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 
 /*-=====  Pre-defined compression levels  =====-*/
 
-#define ZSTD_DEFAULT_CLEVEL 1
 #define ZSTD_MAX_CLEVEL     22
 int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
 
@@ -3592,7 +3712,7 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
     size_t const addedSize = srcSize ? 0 : 500;
     U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1;
     U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);   /* intentional underflow for srcSizeHint == 0 */
-    if (compressionLevel <= 0) compressionLevel = ZSTD_DEFAULT_CLEVEL;   /* 0 == default; no negative compressionLevel yet */
+    if (compressionLevel <= 0) compressionLevel = 1;   /* 0 == default; no negative compressionLevel yet */
     if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
     cp = ZSTD_defaultCParameters[tableID][compressionLevel];
     if (MEM_32bits()) {   /* auto-correction, for 32-bits mode */
diff --git a/lib/zstd.h b/lib/zstd.h
index fcefc7af6..587f9f521 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -588,7 +588,9 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
 
 typedef enum {
     /* compression parameters */
-    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table (default:3) */
+    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
+                              * Default level is 3.
+                              * Special: value 0 means "do not change cLevel". */
     ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
                               * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
                               * default value : set through compressionLevel */
@@ -605,7 +607,7 @@ typedef enum {
                               * More attempts result in better and slower compression.
                               * This parameter is useless when using "fast" and "dFast" strategies */
     ZSTD_p_minMatchLength,   /* Minimum match size (except for repeat-matches, which limit is hard-coded).
-                              * Larger values make compression and decompression faster, but decrease compression ratio
+                              * Larger values make faster compression and decompression, but decrease ratio.
                               * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
                               * Note that currently, for all strategies < btopt, effective minimum is 4.
                               * Note that currently, for all strategies > fast, effective maximum is 6. */
@@ -613,7 +615,7 @@ typedef enum {
                               * Length of Match considered "good enough" to stop search.
                               * Larger values make compression stronger and slower. */
     ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
-                              * Cast selected strategy into unsigned for ZSTD_CCtx_setParameter() compatibility.
+                              * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
                               * The higher the value of selected strategy, the more complex it is,
                               * resulting in stronger and slower compression */
 #if 0

From add66f816d5c4f3f73fe3803578b9ee3820371a0 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 12 May 2017 15:59:48 -0700
Subject: [PATCH 005/109] changed macro LOADCPARAMS by static function
 ZSTD_cLevelToCParams()

for improved compiler checks.
Also : ensure most parameters can receive value "0"
to mean "do not change".
---
 doc/zstd_manual.html         | 20 ++++++++++++------
 lib/compress/zstd_compress.c | 41 ++++++++++++++++++++++++------------
 lib/zstd.h                   | 20 ++++++++++++------
 3 files changed, 53 insertions(+), 28 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 68efb6cc8..8c74bcff0 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -479,31 +479,37 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
                               * Special: value 0 means "do not change cLevel". */
     ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
                               * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
-                              * default value : set through compressionLevel */
+                              * default value : set through compressionLevel.
+                              * Special: value 0 means "do not change windowLog". */
     ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
                               * Resulting table size is (1 << (hashLog+2)).
                               * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
                               * Larger tables improve compression ratio of strategies <= dFast,
-                              * and improve speed of strategies > dFast */
+                              * and improve speed of strategies > dFast.
+                              * Special: value 0 means "do not change hashLog". */
     ZSTD_p_chainLog,         /* Size of the full-search table, as a power of 2.
                               * Resulting table size is (1 << (chainLog+2)).
                               * Larger tables result in better and slower compression.
-                              * This parameter is useless when using "fast" strategy */
+                              * This parameter is useless when using "fast" strategy.
+                              * Special: value 0 means "do not change chainLog". */
     ZSTD_p_searchLog,        /* Number of search attempts, as a power of 2.
                               * More attempts result in better and slower compression.
-                              * This parameter is useless when using "fast" and "dFast" strategies */
+                              * This parameter is useless when using "fast" and "dFast" strategies.
+                              * Special: value 0 means "do not change searchLog". */
     ZSTD_p_minMatchLength,   /* Minimum match size (except for repeat-matches, which limit is hard-coded).
                               * Larger values make faster compression and decompression, but decrease ratio.
                               * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
                               * Note that currently, for all strategies < btopt, effective minimum is 4.
-                              * Note that currently, for all strategies > fast, effective maximum is 6. */
+                              * Note that currently, for all strategies > fast, effective maximum is 6.
+                              * Special: value 0 means "do not change minMatchLength". */
     ZSTD_p_targetLength,     /* Only useful for strategies >= btopt.
                               * Length of Match considered "good enough" to stop search.
-                              * Larger values make compression stronger and slower. */
+                              * Larger values make compression stronger and slower.
+                              * Special: value 0 means "do not change targetLength". */
     ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
                               * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
                               * The higher the value of selected strategy, the more complex it is,
-                              * resulting in stronger and slower compression */
+                              * resulting in stronger and slower compression. */
 #if 0
     ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
                               * Can be set to a more precise value than windowLog.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e17521565..754bb8b08 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -202,15 +202,23 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
     }
 }
 
+
+static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
+{
+    if (cctx->compressionLevel==0)  return;
+    cctx->params.cParams = ZSTD_getCParams(cctx->compressionLevel,
+                                           cctx->frameContentSize, 0);
+    cctx->compressionLevel = 0;
+}
+
 size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
 {
 #   define CLAMPCHECK(val,min,max) { if ((valmax)) return ERROR(compressionParameter_unsupported); }
-#   define LOADCPARAMS(cParams) \
-            if (cctx->compressionLevel!=0)  {               \
-                cParams = ZSTD_getCParams(                  \
-                    cctx->compressionLevel,                 \
-                    cctx->frameContentSize, 0);   /* dictSize unknown at this stage */ \
-                cctx->compressionLevel = 0;                 \
+#   define LOADCPARAMS(cctx)                                        \
+            if (cctx->compressionLevel!=0)  {                          \
+                cctx->params.cParams = ZSTD_getCParams( cctx->compressionLevel,     \
+                                           cctx->frameContentSize, 0); \
+                cctx->compressionLevel = 0;                            \
             }
 
 
@@ -223,44 +231,50 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
             return 0;
 
     case ZSTD_p_windowLog :
+            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
-            LOADCPARAMS(cctx->params.cParams);
+            ZSTD_cLevelToCParams(cctx);
             cctx->params.cParams.windowLog = value;
             return 0;
 
     case ZSTD_p_hashLog :
+            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
-            LOADCPARAMS(cctx->params.cParams);
+            ZSTD_cLevelToCParams(cctx);
             cctx->params.cParams.hashLog = value;
             return 0;
 
     case ZSTD_p_chainLog :
+            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
-            LOADCPARAMS(cctx->params.cParams);
+            ZSTD_cLevelToCParams(cctx);
             cctx->params.cParams.chainLog = value;
             return 0;
 
     case ZSTD_p_searchLog :
+            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
-            LOADCPARAMS(cctx->params.cParams);
+            ZSTD_cLevelToCParams(cctx);
             cctx->params.cParams.searchLog = value;
             return 0;
 
     case ZSTD_p_minMatchLength :
+            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
-            LOADCPARAMS(cctx->params.cParams);
+            ZSTD_cLevelToCParams(cctx);
             cctx->params.cParams.searchLength = value;
             return 0;
 
     case ZSTD_p_targetLength :
+            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
-            LOADCPARAMS(cctx->params.cParams);
+            ZSTD_cLevelToCParams(cctx);
             cctx->params.cParams.targetLength = value;
             return 0;
 
     case ZSTD_p_compressionStrategy :
             if (value > (unsigned)ZSTD_btultra) return ERROR(compressionParameter_unsupported);
-            LOADCPARAMS(cctx->params.cParams);
+            ZSTD_cLevelToCParams(cctx);
             cctx->params.cParams.strategy = (ZSTD_strategy)value;
             return 0;
 
@@ -325,7 +339,6 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
     @return : 0, or an error code if one value is beyond authorized range */
 size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
 {
-#   define CLAMPCHECK(val,min,max) { if ((valmax)) return ERROR(compressionParameter_unsupported); }
     CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
     CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
     CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
diff --git a/lib/zstd.h b/lib/zstd.h
index 587f9f521..c1ae9da54 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -593,31 +593,37 @@ typedef enum {
                               * Special: value 0 means "do not change cLevel". */
     ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
                               * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
-                              * default value : set through compressionLevel */
+                              * default value : set through compressionLevel.
+                              * Special: value 0 means "do not change windowLog". */
     ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
                               * Resulting table size is (1 << (hashLog+2)).
                               * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
                               * Larger tables improve compression ratio of strategies <= dFast,
-                              * and improve speed of strategies > dFast */
+                              * and improve speed of strategies > dFast.
+                              * Special: value 0 means "do not change hashLog". */
     ZSTD_p_chainLog,         /* Size of the full-search table, as a power of 2.
                               * Resulting table size is (1 << (chainLog+2)).
                               * Larger tables result in better and slower compression.
-                              * This parameter is useless when using "fast" strategy */
+                              * This parameter is useless when using "fast" strategy.
+                              * Special: value 0 means "do not change chainLog". */
     ZSTD_p_searchLog,        /* Number of search attempts, as a power of 2.
                               * More attempts result in better and slower compression.
-                              * This parameter is useless when using "fast" and "dFast" strategies */
+                              * This parameter is useless when using "fast" and "dFast" strategies.
+                              * Special: value 0 means "do not change searchLog". */
     ZSTD_p_minMatchLength,   /* Minimum match size (except for repeat-matches, which limit is hard-coded).
                               * Larger values make faster compression and decompression, but decrease ratio.
                               * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
                               * Note that currently, for all strategies < btopt, effective minimum is 4.
-                              * Note that currently, for all strategies > fast, effective maximum is 6. */
+                              * Note that currently, for all strategies > fast, effective maximum is 6.
+                              * Special: value 0 means "do not change minMatchLength". */
     ZSTD_p_targetLength,     /* Only useful for strategies >= btopt.
                               * Length of Match considered "good enough" to stop search.
-                              * Larger values make compression stronger and slower. */
+                              * Larger values make compression stronger and slower.
+                              * Special: value 0 means "do not change targetLength". */
     ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
                               * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
                               * The higher the value of selected strategy, the more complex it is,
-                              * resulting in stronger and slower compression */
+                              * resulting in stronger and slower compression. */
 #if 0
     ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
                               * Can be set to a more precise value than windowLog.

From a5ffe3d37031e5bac9a3a8fd1ea073ced06a9756 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 12 May 2017 16:29:19 -0700
Subject: [PATCH 006/109] pushed enum values for strategy by one (ZSTD_fast==1)

this makes it possible to use `0` to mean:
"do not change strategy"
---
 doc/zstd_manual.html         | 48 ++++++++++++++++++++----------------
 lib/compress/zstd_compress.c | 12 ++++++---
 lib/zstd.h                   | 48 ++++++++++++++++++++----------------
 programs/fileio.c            |  2 +-
 tests/fuzzer.c               |  8 +++---
 tests/zstreamtest.c          |  8 +++---
 6 files changed, 71 insertions(+), 55 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 8c74bcff0..13eb5523e 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -31,9 +31,10 @@
 
 

 
Introduction
-  zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
-  at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
-  decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
+  zstd, short for Zstandard, is a fast lossless compression algorithm,
+  targeting real-time compression scenarios at zlib-level and better compression ratios.
+  The zstd compression library provides in-memory compression and decompression functions.
+  The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
   Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
   Compression can be done in:
     - a single step (described as Simple API)
@@ -73,21 +74,17 @@
 


 
 
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
-
  NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
-  ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
-  frame, but distinguishes empty frames from frames with an unknown size, or errors.
-
-  Additionally, ZSTD_findDecompressedSize can be used instead.  It can handle multiple
-  concatenated frames in one buffer, and so is more general.
-  As a result however, it requires more computation and entire frames to be passed to it,
-  as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
+
  NOTE: This function is planned to be obsolete, in favor of ZSTD_getFrameContentSize().
+  ZSTD_getFrameContentSize() works the same way,
+  returning the decompressed size of a single frame,
+  but distinguishes empty frames from frames with an unknown size, or errors.
 
   'src' is the start of a zstd compressed frame.
   @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
-   note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+   note 1 : decompressed size is an optional field, it may not be present, typically in streaming mode.
             When `return==0`, data to decompress could be any size.
             In which case, it's necessary to use streaming mode to decompress data.
-            Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+            Optionally, application can use ZSTD_decompress() while relying on implied limits.
             (For example, data may be necessarily cut into blocks <= 16 KB).
    note 2 : decompressed size is always present when compression is done with ZSTD_compress()
    note 3 : decompressed size can be very large (64-bits value),
@@ -114,20 +111,26 @@ const char* ZSTD_getErrorName(size_t code);     /*!< provides readable strin
 ZSTD_CCtx* ZSTD_createCCtx(void);
 size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 


-
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
+
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
+                         void* dst, size_t dstCapacity,
+                   const void* src, size_t srcSize,
+                         int compressionLevel);
 
  Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 


 
 
Decompression context
  When decompressing many times,
-  it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+  it is recommended to allocate a context only once,
+  and re-use it for each successive compression operation.
   This will make workload friendlier for system's memory.
-  Use one context per thread for parallel execution in multi-threaded environments. 
+  Use one context per thread for parallel execution. 
 
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 


-
size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-
  Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). 
+
size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
+                           void* dst, size_t dstCapacity,
+                     const void* src, size_t srcSize);
+
  Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) 
 


 
 
Simple dictionary API

@@ -154,7 +157,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 
Fast dictionary API

 
-
ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
+                             int compressionLevel);
 
   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
    ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
    ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
@@ -298,7 +302,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
 
Advanced types

 
-
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy;   /* from faster to stronger */
+
typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
+               ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy;   /* from faster to stronger */
 


 
typedef struct {
     unsigned windowLog;      /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -509,7 +514,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
     ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
                               * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
                               * The higher the value of selected strategy, the more complex it is,
-                              * resulting in stronger and slower compression. */
+                              * resulting in stronger and slower compression.
+                              * Special: value 0 means "do not change strategy". */
 #if 0
     ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
                               * Can be set to a more precise value than windowLog.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 754bb8b08..565cf04a1 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -273,7 +273,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
             return 0;
 
     case ZSTD_p_compressionStrategy :
-            if (value > (unsigned)ZSTD_btultra) return ERROR(compressionParameter_unsupported);
+            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+            CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
             ZSTD_cLevelToCParams(cctx);
             cctx->params.cParams.strategy = (ZSTD_strategy)value;
             return 0;
@@ -2575,14 +2576,17 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr
 
 static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
 {
-    static const ZSTD_blockCompressor blockCompressor[2][8] = {
-        { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy,
+    static const ZSTD_blockCompressor blockCompressor[2][(unsigned)ZSTD_btultra+1] = {
+        { NULL,
+          ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy,
           ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2,
           ZSTD_compressBlock_btopt, ZSTD_compressBlock_btultra },
-        { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict,
+        { NULL,
+          ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict,
           ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict,
           ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict }
     };
+    ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
 
     return blockCompressor[extDict][(U32)strat];
 }
diff --git a/lib/zstd.h b/lib/zstd.h
index c1ae9da54..b6e4b263c 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -36,9 +36,10 @@ extern "C" {
 /*******************************************************************************************************
   Introduction
 
-  zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
-  at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
-  decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
+  zstd, short for Zstandard, is a fast lossless compression algorithm,
+  targeting real-time compression scenarios at zlib-level and better compression ratios.
+  The zstd compression library provides in-memory compression and decompression functions.
+  The library supports compression levels from 1 up to ZSTD_maxCLevel() which is currently 22.
   Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
   Compression can be done in:
     - a single step (described as Simple API)
@@ -89,21 +90,17 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                               const void* src, size_t compressedSize);
 
 /*! ZSTD_getDecompressedSize() :
- *  NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
- *  ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
- *  frame, but distinguishes empty frames from frames with an unknown size, or errors.
- *
- *  Additionally, ZSTD_findDecompressedSize can be used instead.  It can handle multiple
- *  concatenated frames in one buffer, and so is more general.
- *  As a result however, it requires more computation and entire frames to be passed to it,
- *  as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
+ *  NOTE: This function is planned to be obsolete, in favor of ZSTD_getFrameContentSize().
+ *  ZSTD_getFrameContentSize() works the same way,
+ *  returning the decompressed size of a single frame,
+ *  but distinguishes empty frames from frames with an unknown size, or errors.
  *
  *  'src' is the start of a zstd compressed frame.
  *  @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
- *   note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+ *   note 1 : decompressed size is an optional field, it may not be present, typically in streaming mode.
  *            When `return==0`, data to decompress could be any size.
  *            In which case, it's necessary to use streaming mode to decompress data.
- *            Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+ *            Optionally, application can use ZSTD_decompress() while relying on implied limits.
  *            (For example, data may be necessarily cut into blocks <= 16 KB).
  *   note 2 : decompressed size is always present when compression is done with ZSTD_compress()
  *   note 3 : decompressed size can be very large (64-bits value),
@@ -137,20 +134,26 @@ ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 
 /*! ZSTD_compressCCtx() :
  *  Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
-ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx,
+                                     void* dst, size_t dstCapacity,
+                               const void* src, size_t srcSize,
+                                     int compressionLevel);
 
 /*= Decompression context
  *  When decompressing many times,
- *  it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+ *  it is recommended to allocate a context only once,
+ *  and re-use it for each successive compression operation.
  *  This will make workload friendlier for system's memory.
- *  Use one context per thread for parallel execution in multi-threaded environments. */
+ *  Use one context per thread for parallel execution. */
 typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 /*! ZSTD_decompressDCtx() :
- *  Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */
-ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ *  Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
+ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
+                                       void* dst, size_t dstCapacity,
+                                 const void* src, size_t srcSize);
 
 
 /**************************
@@ -187,7 +190,8 @@ typedef struct ZSTD_CDict_s ZSTD_CDict;
 *   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
 *   ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
 *   `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
+                                         int compressionLevel);
 
 /*! ZSTD_freeCDict() :
 *   Function frees memory allocated by ZSTD_createCDict(). */
@@ -375,7 +379,8 @@ static const size_t ZSTD_skippableHeaderSize = 8;  /* magic number + skippable f
 
 
 /*--- Advanced types ---*/
-typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy;   /* from faster to stronger */
+typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
+               ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy;   /* from faster to stronger */
 
 typedef struct {
     unsigned windowLog;      /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -623,7 +628,8 @@ typedef enum {
     ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
                               * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
                               * The higher the value of selected strategy, the more complex it is,
-                              * resulting in stronger and slower compression. */
+                              * resulting in stronger and slower compression.
+                              * Special: value 0 means "do not change strategy". */
 #if 0
     ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
                               * Can be set to a more precise value than windowLog.
diff --git a/programs/fileio.c b/programs/fileio.c
index ba15555dd..2c12b924c 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -369,7 +369,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
             if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
             if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
             if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
-            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);   /* 0 means : do not change */
+            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
 #ifdef ZSTD_MULTITHREAD
             {   size_t const errorCode = ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
                 if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 4b9cd97a5..6814d5476 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -12,9 +12,9 @@
 *  Compiler specific
 **************************************/
 #ifdef _MSC_VER    /* Visual Studio */
-#  define _CRT_SECURE_NO_WARNINGS     /* fgets */
-#  pragma warning(disable : 4127)     /* disable: C4127: conditional expression is constant */
-#  pragma warning(disable : 4204)     /* disable: C4204: non-constant aggregate initializer */
+#  define _CRT_SECURE_NO_WARNINGS   /* fgets */
+#  pragma warning(disable : 4127)   /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4204)   /* disable: C4204: non-constant aggregate initializer */
 #endif
 
 
@@ -25,7 +25,7 @@
 #include         /* fgets, sscanf */
 #include        /* strcmp */
 #include          /* clock_t */
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressContinue, ZSTD_compressBlock */
+#define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_compressContinue, ZSTD_compressBlock */
 #include "zstd.h"         /* ZSTD_VERSION_STRING */
 #include "zstd_errors.h"  /* ZSTD_getErrorCode */
 #include "zstdmt_compress.h"
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 5c166bfb8..6c4900ebb 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -12,9 +12,9 @@
 *  Compiler specific
 **************************************/
 #ifdef _MSC_VER    /* Visual Studio */
-#  define _CRT_SECURE_NO_WARNINGS     /* fgets */
-#  pragma warning(disable : 4127)     /* disable: C4127: conditional expression is constant */
-#  pragma warning(disable : 4146)     /* disable: C4146: minus unsigned expression */
+#  define _CRT_SECURE_NO_WARNINGS   /* fgets */
+#  pragma warning(disable : 4127)   /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4146)   /* disable: C4146: minus unsigned expression */
 #endif
 
 
@@ -26,7 +26,7 @@
 #include          /* clock_t, clock() */
 #include        /* strcmp */
 #include "mem.h"
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
+#define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
 #include "zstd.h"         /* ZSTD_compressBound */
 #include "zstd_errors.h"  /* ZSTD_error_srcSize_wrong */
 #include "zstdmt_compress.h"

From 9f95e445ab839b0927ccdfb5518e6143deb89f59 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 16 May 2017 17:26:43 -0700
Subject: [PATCH 007/109] minor comment clarifications

---
 lib/zstd.h | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index 116569b43..e8933f9d7 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -567,14 +567,14 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
 /*===   New experimental advanced parameters API   ===*/
 
 /* notes on API design :
- *   In below proposal, parameters are pushed one by one into an existing CCtx,
+ *   In this proposal, parameters are pushed one by one into an existing CCtx,
  *   and then applied on all following compression jobs.
  *   When no parameter is ever provided, CCtx is created with compression level 3.
  *
  *   Another approach could be to load parameters into an intermediate _opaque_ object.
  *   The object would then be loaded into CCtx (like ZSTD_compress_advanced())
  *   This approach would be compatible with ZSTD_createCDict_advanced().
- *   But it's a bit more cumbersome for CCtx, because it requires to manage an additional object.
+ *   But it's more cumbersome for CCtx, as it requires to manage an additional object.
  *
  *   Below proposal push parameters directly into CCtx.
  *   It's a bit more complex for CDict, as advanced version now requires 3 steps.
@@ -583,13 +583,13 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
  */
 
 /* notes on naming convention :
- *   Initially, the API favored naming like ZSTD_setCCtxParameter() .
- *   In this proposal, this is changed into ZSTD_CCtx_setParameter() .
+ *   Initially, the API favored names like ZSTD_setCCtxParameter() .
+ *   In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
  *   The main idea is that it identifies more clearly the target object type.
  *   It feels clearer when considering other potential variants :
  *   ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
  *   ZSTD_DCtx_setParameter()  (rather than ZSTD_setDCtxParameter() )
- *   The first verion looks clearer.
+ *   The left variant looks clearer.
  */
 
 typedef enum {
@@ -673,7 +673,8 @@ typedef enum {
 
 /*! ZSTD_CCtx_setParameter() :
  *  Set one compression parameter, selected by enum ZSTD_cParameter.
- *  @result : 0, or an error code (which can be tested with ZSTD_isError()) */
+ *  Note : when `value` is an enum, cast it to unsigned for proper type checking.
+ *  @result : 0, or an error code (which can be tested with ZSTD_isError()). */
 ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
 
 /*! ZSTD_CCtx_setPledgedSrcSize() :
@@ -719,7 +720,7 @@ ZSTDLIB_API size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict
  *  Add a prepared dictionary to cctx, it will used for next compression jobs.
  *  Note that compression parameters will be enforced from within CDict.
  *  Currently, they supercede any compression parameter previously set within CCtx.
- *  The dictionary will remain valid for all future compression jobs performed using the same cctx.
+ *  The dictionary will remain valid for future compression jobs using same cctx.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special : adding a NULL CDict means "return to no-dictionary mode".
  *  Note 1 : Currently, only one dictionary can be managed.
@@ -734,7 +735,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 
 typedef enum {
     ZSTD_e_continue,  /* continue sending data, encoder transparently decides when to output result, depending on optimal conditions */
-    ZSTD_e_flush,     /* flush any data provided and buffered so far - frame will continue, future data can still reference previous data for better compression */
+    ZSTD_e_flush,     /* flush any data provided, compressed and buffered so far - frame will continue, future data can still reference previous data for better compression */
     ZSTD_e_end        /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
 } ZSTD_EndDirective;
 
@@ -744,14 +745,15 @@ typedef enum {
  *  - Compression parameters cannot be changed once compression is started.
  *  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
  *  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
- *  - @return provides the amount of data ready to flush and still within internal buffers
+ *  - @return provides the amount of data ready to flush within internal buffers
  *            or an error code, which can be tested using ZSTD_isError().
- *            if @return != 0, flush is not fully completed, so it must be called again to empty internal buffers.
+ *            if @return != 0, flush is not fully completed, and must be called again to empty internal buffers.
  *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
  *            only ZSTD_e_end and ZSTD_e_flush operations are allowed.
  *            It is necessary to fully flush internal buffers
  *            before changing compression parameters or start a new compression job.
  */
+// Not ready yet !!!
 ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
                                           void* dst, size_t dstCapacity, size_t* dstPos,
                                     const void* src, size_t srcSize, size_t* srcPos,
@@ -763,7 +765,6 @@ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
  *  It's allowed to change compression parameters after a reset.
  *  Any internal data not yet flushed is cancelled.
  */
-// Not ready yet !!!
 ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
 
 

From 23c256e44b4a7216d09c8070d02d690ec8912ae2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 16 May 2017 18:10:11 -0700
Subject: [PATCH 008/109] removed useless variable from CCtx

CStream's pledgedSrcSize is no longer necessary
srcSize control is realized within bufferless interface.
---
 lib/compress/zstd_compress.c |  2 --
 lib/zstd.h                   |  4 ++--
 programs/fileio.c            | 34 ++++++++++++++++++++--------------
 3 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 565cf04a1..3d1236d05 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -140,7 +140,6 @@ struct ZSTD_CCtx_s {
     size_t outBuffFlushedSize;
     ZSTD_cStreamStage streamStage;
     U32    frameEnded;
-    U64    pledgedSrcSize;
 };
 
 ZSTD_CCtx* ZSTD_createCCtx(void)
@@ -3320,7 +3319,6 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters para
     zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
     zcs->streamStage = zcss_load;
     zcs->frameEnded = 0;
-    zcs->pledgedSrcSize = pledgedSrcSize;
     return 0;   /* ready to go */
 }
 
diff --git a/lib/zstd.h b/lib/zstd.h
index e8933f9d7..4d14c8026 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -647,7 +647,7 @@ typedef enum {
     /* dictionary parameters */
     ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
                               * This avoids duplicating dictionary content.
-                              * But it also requires that dictionary buffer outlives its user (CCtx or CDict) */
+                              * But it also requires that dictionary buffer outlives its user (CDict) */
                              /* Not ready yet ! */
     ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
                              /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
@@ -717,7 +717,7 @@ ZSTDLIB_API size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter pa
 ZSTDLIB_API size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
 
 /*! ZSTD_CCtx_refCDict() :
- *  Add a prepared dictionary to cctx, it will used for next compression jobs.
+ *  Add a prepared dictionary to cctx, to be used for next compression jobs.
  *  Note that compression parameters will be enforced from within CDict.
  *  Currently, they supercede any compression parameter previously set within CCtx.
  *  The dictionary will remain valid for future compression jobs using same cctx.
diff --git a/programs/fileio.c b/programs/fileio.c
index 44152b0b1..a36abcaff 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -77,7 +77,7 @@
 #define BLOCKSIZE      (128 KB)
 #define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
 
-#define FIO_FRAMEHEADERSIZE  5        /* as a define, because needed to allocated table on stack */
+#define FIO_FRAMEHEADERSIZE  5    /* as a define, because needed to allocated table on stack */
 #define FSE_CHECKSUM_SEED    0
 
 #define CACHELINE 64
@@ -92,7 +92,7 @@
 ***************************************/
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
-static int g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result + interaction + warnings;   3 : + progression;   4 : + information */
+static int g_displayLevel = 2;   /* 0 : no display;  1: errors;  2: + result + interaction + warnings;  3: + progression;  4: + information */
 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 
 #define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \
@@ -102,7 +102,7 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
 static clock_t g_time = 0;
 
-#undef MIN
+#undef MIN  /* in case it would be already defined */
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
 
@@ -226,11 +226,13 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
         SET_BINARY_MODE(stdin);
     } else {
         if (!UTIL_isRegFile(srcFileName)) {
-            DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName);
+            DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
+                            srcFileName);
             return NULL;
         }
         f = fopen(srcFileName, "rb");
-        if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
+        if ( f==NULL )
+            DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
     }
 
     return f;
@@ -255,26 +257,28 @@ static FILE* FIO_openDstFile(const char* dstFileName)
         if (g_sparseFileSupport == 1) {
             g_sparseFileSupport = ZSTD_SPARSE_DEFAULT;
         }
-        if (strcmp (dstFileName, nulmark)) {  /* Check if destination file already exists */
+        if (strcmp (dstFileName, nulmark)) {
+            /* Check if destination file already exists */
             f = fopen( dstFileName, "rb" );
-            if (f != 0) {  /* dest file exists, prompt for overwrite authorization */
+            if (f != 0) {  /* dst file exists, prompt for overwrite authorization */
                 fclose(f);
                 if (!g_overwrite) {
                     if (g_displayLevel <= 1) {
                         /* No interaction possible */
-                        DISPLAY("zstd: %s already exists; not overwritten  \n", dstFileName);
+                        DISPLAY("zstd: %s already exists; not overwritten  \n",
+                                dstFileName);
                         return NULL;
                     }
-                    DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
+                    DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ",
+                            dstFileName);
                     {   int ch = getchar();
                         if ((ch!='Y') && (ch!='y')) {
                             DISPLAY("    not overwritten  \n");
                             return NULL;
                         }
-                        while ((ch!=EOF) && (ch!='\n')) ch = getchar();  /* flush rest of input line */
-                    }
-                }
-
+                        /* flush rest of input line */
+                        while ((ch!=EOF) && (ch!='\n')) ch = getchar();
+                }   }
                 /* need to unlink */
                 FIO_remove(dstFileName);
         }   }
@@ -304,7 +308,9 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
     fileHandle = fopen(fileName, "rb");
     if (fileHandle==0) EXM_THROW(31, "zstd: %s: %s", fileName, strerror(errno));
     fileSize = UTIL_getFileSize(fileName);
-    if (fileSize > DICTSIZE_MAX) EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", fileName, DICTSIZE_MAX >> 20);   /* avoid extreme cases */
+    if (fileSize > DICTSIZE_MAX)
+        EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
+                        fileName, DICTSIZE_MAX >> 20);   /* avoid extreme cases */
     *bufferPtr = malloc((size_t)fileSize);
     if (*bufferPtr==NULL) EXM_THROW(34, "zstd: %s", strerror(errno));
     { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);

From 6d4fef36de21908e333b2a1fde8ded0a7f086ae1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 17 May 2017 18:36:15 -0700
Subject: [PATCH 009/109] Added ZSTD_compress_generic()

Used in fileio.c (zstd cli).
Need to set macro ZSTD_NEWAPI to trigger it.
---
 doc/zstd_manual.html         |  67 ++++++++-----
 lib/compress/zstd_compress.c | 126 +++++++++++++++++------
 lib/zstd.h                   |  44 +++++---
 programs/fileio.c            | 189 +++++++++++++++++++++--------------
 programs/util.h              |   2 +-
 5 files changed, 277 insertions(+), 151 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 13eb5523e..ff1ecd92f 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -51,7 +51,7 @@
 
 
Version

 
-
unsigned ZSTD_versionNumber(void);   /**< library version number; to be used when checking dll version */
+
unsigned ZSTD_versionNumber(void);   /**< to be used when checking dll version */
 


 
Simple API

 
@@ -501,7 +501,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
                               * More attempts result in better and slower compression.
                               * This parameter is useless when using "fast" and "dFast" strategies.
                               * Special: value 0 means "do not change searchLog". */
-    ZSTD_p_minMatchLength,   /* Minimum match size (except for repeat-matches, which limit is hard-coded).
+    ZSTD_p_minMatch,         /* Minimum size of searched matches (note : repCode matches can be smaller).
                               * Larger values make faster compression and decompression, but decrease ratio.
                               * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
                               * Note that currently, for all strategies < btopt, effective minimum is 4.
@@ -526,13 +526,13 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
 
     /* frame parameters */
     ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
-    ZSTD_p_contentChecksumFlag, /* A 32-bits content checksum is calculated and written at end of frame (default:0) */
+    ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
 
     /* dictionary parameters */
     ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
                               * This avoids duplicating dictionary content.
-                              * But it also requires that dictionary buffer outlives its user (CCtx or CDict) */
+                              * But it also requires that dictionary buffer outlives its user (CDict) */
                              /* Not ready yet ! */
     ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
                              /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
@@ -557,7 +557,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
 


 
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
 
  Set one compression parameter, selected by enum ZSTD_cParameter.
-  @result : 0, or an error code (which can be tested with ZSTD_isError()) 
+  Note : when `value` is an enum, cast it to unsigned for proper type checking.
+  @result : 0, or an error code (which can be tested with ZSTD_isError()). 
 


 
 
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
@@ -571,17 +572,32 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
            this value is overriden by srcSize instead. 
 


 
+
size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);   /* Not ready yet ! */
+
  Create an internal CDict from dict buffer.
+  Decompression will have to use same buffer.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix,
+            meaning "return to no-dictionary mode".
+  Note 1 : Dictionary content will be copied internally,
+           except if ZSTD_p_refDictContent is set.
+  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
+           For this reason, compression parameters cannot be changed anymore after loading a prefix.
+           It's also a CPU-heavy operation, with non-negligible impact on latency.
+  Note 3 : Dictionary will be used for all future compression jobs.
+           To return to "no-dictionary" situation, load a NULL dictionary 
+


+
 
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
 
  Reference a prefix (content-only dictionary) to bootstrap next compression job.
   Decompression will have to use same prefix.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
-  Note 1 : Prefix content is referenced. It must outlive compression job.
+  Note 1 : Prefix buffer is referenced. It must outlive compression job.
+  Note 3 : Prefix is only used once. Tables are discarded at end of compression job.
+           If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
   Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
            For this reason, compression parameters cannot be changed anymore after loading a prefix.
-           It's also a CPU-heavy operation, with non-negligible impact on latency.
-  Note 3 : Prefix is only used once. Tables are discarded at end of compression job.
-           If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict 
+           It's also a CPU-heavy operation, with non-negligible impact on latency. 
 


 
 
ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
@@ -599,10 +615,10 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
 


 
 
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);  /* Not ready yet ! */
-
  Add a prepared dictionary to cctx, it will used for next compression jobs.
+
  Add a prepared dictionary to cctx, to be used for next compression jobs.
   Note that compression parameters will be enforced from within CDict.
   Currently, they supercede any compression parameter previously set within CCtx.
-  The dictionary will remain valid for all future compression jobs performed using the same cctx.
+  The dictionary will remain valid for future compression jobs using same cctx.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special : adding a NULL CDict means "return to no-dictionary mode".
   Note 1 : Currently, only one dictionary can be managed.
@@ -612,23 +628,23 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
 


 
 
typedef enum {
-    ZSTD_e_continue,  /* continue sending data, encoder transparently decides when to output result, depending on optimal conditions */
-    ZSTD_e_flush,     /* flush any data provided and buffered so far - frame will continue, future data can still reference previous data for better compression */
-    ZSTD_e_end        /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
+    ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
+    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
+    ZSTD_e_end         /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
 } ZSTD_EndDirective;
 


 
size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
-                              void* dst, size_t dstCapacity, size_t* dstPos,
-                        const void* src, size_t srcSize, size_t* srcPos,
+                              ZSTD_outBuffer* output,
+                              ZSTD_inBuffer* input,
                               ZSTD_EndDirective endOp);
 
  Behave about the same as ZSTD_compressStream. To note :
   - Compression parameters are pushed into CCtx before starting compression, using ZSTD_setCCtxParameter()
   - Compression parameters cannot be changed once compression is started.
   - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
   - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
-  - @return provides the amount of data ready to flush and still within internal buffers
+  - @return provides the amount of data ready to flush within internal buffers
             or an error code, which can be tested using ZSTD_isError().
-            if @return != 0, flush is not fully completed, so it must be called again to empty internal buffers.
+            if @return != 0, flush is not fully completed, and must be called again to empty internal buffers.
   - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
             only ZSTD_e_end and ZSTD_e_flush operations are allowed.
             It is necessary to fully flush internal buffers
@@ -636,11 +652,10 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
  
 


 
-
// Not ready yet !!!
-size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
+
size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
 
  Return a CCtx to clean state.
   Useful after an error, or to interrupt an ongoing compression job and start a new one.
-  It's allowed to change compression parameters after a reset.
+  It's possible to modify compression parameters after a reset.
   Any internal data not yet flushed is cancelled.
  
 


@@ -781,7 +796,8 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
            >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
-  Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
+  Start decompression, with ZSTD_decompressBegin().
+  If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
   Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
 
   Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
@@ -823,12 +839,11 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
 
Buffer-less streaming decompression functions
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
+size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
 void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
-size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
-size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
-ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
 


+
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
+


 
Block functions
     Block functions produce and decode raw zstd blocks, without frame metadata.
     Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3d1236d05..1d758dab7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -11,8 +11,8 @@
 /*-*************************************
 *  Tuning parameters
 ***************************************/
-#ifndef ZSTD_DEFAULT_CLEVEL
-#  define ZSTD_DEFAULT_CLEVEL 3
+#ifndef ZSTD_CLEVEL_DEFAULT
+#  define ZSTD_CLEVEL_DEFAULT 3
 #endif
 
 /*-*************************************
@@ -86,7 +86,7 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
 /*-*************************************
 *  Context memory management
 ***************************************/
-typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
+typedef enum { zcss_init=0, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
 
 struct ZSTD_CCtx_s {
     const BYTE* nextSrc;    /* next block here to continue on current prefix */
@@ -158,7 +158,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
     if (!cctx) return NULL;
     memset(cctx, 0, sizeof(ZSTD_CCtx));
     cctx->customMem = customMem;
-    cctx->compressionLevel = ZSTD_DEFAULT_CLEVEL;
+    cctx->compressionLevel = ZSTD_CLEVEL_DEFAULT;
     return cctx;
 }
 
@@ -202,24 +202,18 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
 }
 
 
+#define ZSTD_CLEVEL_CUSTOM 999
 static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
 {
-    if (cctx->compressionLevel==0)  return;
+    if (cctx->compressionLevel==ZSTD_CLEVEL_CUSTOM)  return;
     cctx->params.cParams = ZSTD_getCParams(cctx->compressionLevel,
                                            cctx->frameContentSize, 0);
-    cctx->compressionLevel = 0;
+    cctx->compressionLevel = ZSTD_CLEVEL_CUSTOM;
 }
 
 size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
 {
 #   define CLAMPCHECK(val,min,max) { if ((valmax)) return ERROR(compressionParameter_unsupported); }
-#   define LOADCPARAMS(cctx)                                        \
-            if (cctx->compressionLevel!=0)  {                          \
-                cctx->params.cParams = ZSTD_getCParams( cctx->compressionLevel,     \
-                                           cctx->frameContentSize, 0); \
-                cctx->compressionLevel = 0;                            \
-            }
-
 
     switch(param)
     {
@@ -257,7 +251,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
             cctx->params.cParams.searchLog = value;
             return 0;
 
-    case ZSTD_p_minMatchLength :
+    case ZSTD_p_minMatch :
             if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
             ZSTD_cLevelToCParams(cctx);
@@ -287,7 +281,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
             cctx->params.fParams.contentSizeFlag = value>0;
             return 0;
 
-    case ZSTD_p_contentChecksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
+    case ZSTD_p_checksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
             cctx->params.fParams.checksumFlag = value>0;
             return 0;
 
@@ -319,6 +313,24 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
     return 0;
 }
 
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
+{
+    ZSTD_freeCDict(cctx->cdictLocal);  /* in case one already exists */
+    if (dict==NULL || dictSize==0) {   /* no dictionary mode */
+        cctx->cdictLocal = NULL;
+        cctx->cdict = NULL;
+    } else {
+        cctx->cdictLocal = ZSTD_createCDict_advanced(
+                                dict, dictSize,
+                                0 /* byReference */,
+                                cctx->params.cParams, cctx->customMem);
+        cctx->cdict = cctx->cdictLocal;
+        if (cctx->cdictLocal == NULL)
+            return ERROR(memory_allocation);
+    }
+    return 0;
+}
+
 /* Not ready yet ! */
 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
 {
@@ -3306,8 +3318,6 @@ size_t ZSTD_CStreamOutSize(void)
 
 static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
-    if (zcs->inBuffSize==0) return ERROR(stage_wrong);   /* zcs has not been init at least once => can't reset */
-
     DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag);
 
     if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs, zcs->cdict, params.fParams, pledgedSrcSize))
@@ -3327,7 +3337,9 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 
     ZSTD_parameters params = zcs->params;
     params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
-    DEBUGLOG(5, "ZSTD_resetCStream : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag);
+    if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
+        params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
+    }
     return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
 }
 
@@ -3437,8 +3449,6 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
 
 /*======   Compression   ======*/
 
-typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e;
-
 MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     size_t const length = MIN(dstCapacity, srcSize);
@@ -3449,7 +3459,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
 static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                               void* dst, size_t* dstCapacityPtr,
                         const void* src, size_t* srcSizePtr,
-                              ZSTD_flush_e const flush)
+                              ZSTD_EndDirective const flush)
 {
     U32 someMoreWork = 1;
     const char* const istart = (const char*)src;
@@ -3460,10 +3470,12 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
     char* op = ostart;
 
     DEBUGLOG(5, "ZSTD_compressStream_generic \n");
+    assert(zcs->inBuff != NULL);
+    assert(zcs->outBuff!= NULL);
     while (someMoreWork) {
         switch(zcs->streamStage)
         {
-        case zcss_init: return ERROR(init_missing);   /* call ZBUFF_compressInit() first ! */
+        case zcss_init: return ERROR(init_missing);   /* call ZSTD_initCStream() first ! */
 
         case zcss_load:
             /* complete inBuffer */
@@ -3485,12 +3497,12 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                     cDst = op;   /* compress directly into output buffer (avoid flush stage) */
                 else
                     cDst = zcs->outBuff, oSize = zcs->outBuffSize;
-                cSize = (flush == zsf_end) ?
+                cSize = (flush == ZSTD_e_end) ?
                         ZSTD_compressEnd(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
                         ZSTD_compressContinue(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
                 if (ZSTD_isError(cSize)) return cSize;
                 DEBUGLOG(5, "cSize = %u \n", (U32)cSize);
-                if (flush == zsf_end) zcs->frameEnded = 1;
+                if (flush == ZSTD_e_end) zcs->frameEnded = 1;
                 /* prepare next block */
                 zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
                 if (zcs->inBuffTarget > zcs->inBuffSize)
@@ -3538,14 +3550,60 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf
 {
     size_t sizeRead = input->size - input->pos;
     size_t sizeWritten = output->size - output->pos;
-    size_t const result = ZSTD_compressStream_generic(zcs,
-                                                      (char*)(output->dst) + output->pos, &sizeWritten,
-                                                      (const char*)(input->src) + input->pos, &sizeRead, zsf_gather);
+    size_t const result = ZSTD_compressStream_generic(
+                            zcs,
+                            (char*)(output->dst) + output->pos, &sizeWritten,
+                            (const char*)(input->src) + input->pos, &sizeRead, ZSTD_e_continue);
     input->pos += sizeRead;
     output->pos += sizeWritten;
     return result;
 }
 
+size_t ZSTD_compress_generic_integral (
+                            ZSTD_CCtx* cctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos,
+                            ZSTD_EndDirective endOp)
+{
+    /* check conditions */
+    if (*dstPos > dstCapacity) return ERROR(GENERIC);
+    if (*srcPos > srcSize) return ERROR(GENERIC);
+
+    assert(cctx!=NULL);
+    if (cctx->streamStage == zcss_init) {
+        /* transparent reset */
+        ZSTD_parameters params = cctx->params;
+        if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
+            params.cParams = ZSTD_getCParams(cctx->compressionLevel,
+                                    cctx->frameContentSize, 0 /* dictSize */);
+        DEBUGLOG(5, "starting ZSTD_resetCStream");
+        CHECK_F( ZSTD_initCStream_stage2(cctx, params, cctx->frameContentSize) );
+    }
+
+    {   size_t sizeRead = srcSize - *srcPos;
+        size_t sizeWritten = dstCapacity - *dstPos;
+        DEBUGLOG(5, "starting ZSTD_compressStream_generic");
+        CHECK_F( ZSTD_compressStream_generic(cctx,
+                        (char*)dst + *dstPos, &sizeWritten,
+                        (const char*)src + *srcPos, &sizeRead, endOp) );
+        *srcPos += sizeRead;
+        *dstPos += sizeWritten;
+    }
+    DEBUGLOG(5, "completing ZSTD_compress_generic_integral");
+    return cctx->outBuffContentSize - cctx->outBuffFlushedSize;   /* remaining to flush */
+}
+
+size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
+                              ZSTD_outBuffer* output,
+                              ZSTD_inBuffer* input,
+                              ZSTD_EndDirective endOp)
+{
+    return ZSTD_compress_generic_integral(cctx,
+                output->dst, output->size, &output->pos,
+                input->src, input->size, &input->pos,
+                endOp);
+}
+
 
 /*======   Finalize   ======*/
 
@@ -3556,9 +3614,9 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
     size_t srcSize = 0;
     size_t sizeWritten = output->size - output->pos;
     size_t const result = ZSTD_compressStream_generic(zcs,
-                                                     (char*)(output->dst) + output->pos, &sizeWritten,
-                                                     &srcSize, &srcSize, /* use a valid src address instead of NULL */
-                                                      zsf_flush);
+                                (char*)(output->dst) + output->pos, &sizeWritten,
+                                &srcSize, &srcSize, /* use a valid src address instead of NULL */
+                                ZSTD_e_flush);
     output->pos += sizeWritten;
     if (ZSTD_isError(result)) return result;
     return zcs->outBuffContentSize - zcs->outBuffFlushedSize;   /* remaining to flush */
@@ -3576,8 +3634,10 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
         /* flush whatever remains */
         size_t srcSize = 0;
         size_t sizeWritten = output->size - output->pos;
-        size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten,
-                                     &srcSize /* use a valid src address instead of NULL */, &srcSize, zsf_end);
+        size_t const notEnded = ZSTD_compressStream_generic(zcs,
+                                    ostart, &sizeWritten,
+                                    &srcSize /* valid address */, &srcSize,
+                                    ZSTD_e_end);
         size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
         op += sizeWritten;
         if (remainingToFlush) {
@@ -3727,7 +3787,7 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
     size_t const addedSize = srcSize ? 0 : 500;
     U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1;
     U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);   /* intentional underflow for srcSizeHint == 0 */
-    if (compressionLevel <= 0) compressionLevel = 1;   /* 0 == default; no negative compressionLevel yet */
+    if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* 0 == default; no negative compressionLevel yet */
     if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
     cp = ZSTD_defaultCParameters[tableID][compressionLevel];
     if (MEM_32bits()) {   /* auto-correction, for 32-bits mode */
diff --git a/lib/zstd.h b/lib/zstd.h
index 4d14c8026..338091834 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -616,7 +616,7 @@ typedef enum {
                               * More attempts result in better and slower compression.
                               * This parameter is useless when using "fast" and "dFast" strategies.
                               * Special: value 0 means "do not change searchLog". */
-    ZSTD_p_minMatchLength,   /* Minimum match size (except for repeat-matches, which limit is hard-coded).
+    ZSTD_p_minMatch,         /* Minimum size of searched matches (note : repCode matches can be smaller).
                               * Larger values make faster compression and decompression, but decrease ratio.
                               * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
                               * Note that currently, for all strategies < btopt, effective minimum is 4.
@@ -641,7 +641,7 @@ typedef enum {
 
     /* frame parameters */
     ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
-    ZSTD_p_contentChecksumFlag, /* A 32-bits content checksum is calculated and written at end of frame (default:0) */
+    ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
 
     /* dictionary parameters */
@@ -688,17 +688,32 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param
  *           this value is overriden by srcSize instead. */
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
 
+/*! ZSTD_CCtx_loadDictionary() :
+ *  Create an internal CDict from dict buffer.
+ *  Decompression will have to use same buffer.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix,
+ *            meaning "return to no-dictionary mode".
+ *  Note 1 : Dictionary content will be copied internally,
+ *           except if ZSTD_p_refDictContent is set.
+ *  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
+ *           For this reason, compression parameters cannot be changed anymore after loading a prefix.
+ *           It's also a CPU-heavy operation, with non-negligible impact on latency.
+ *  Note 3 : Dictionary will be used for all future compression jobs.
+ *           To return to "no-dictionary" situation, load a NULL dictionary */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);   /* Not ready yet ! */
+
 /*! ZSTD_CCtx_refPrefix() :
  *  Reference a prefix (content-only dictionary) to bootstrap next compression job.
  *  Decompression will have to use same prefix.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
- *  Note 1 : Prefix content is referenced. It must outlive compression job.
+ *  Note 1 : Prefix buffer is referenced. It must outlive compression job.
+ *  Note 3 : Prefix is only used once. Tables are discarded at end of compression job.
+ *           If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
  *  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
  *           For this reason, compression parameters cannot be changed anymore after loading a prefix.
- *           It's also a CPU-heavy operation, with non-negligible impact on latency.
- *  Note 3 : Prefix is only used once. Tables are discarded at end of compression job.
- *           If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict */
+ *           It's also a CPU-heavy operation, with non-negligible impact on latency. */
 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
 
 
@@ -730,13 +745,11 @@ ZSTDLIB_API size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict
 ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);  /* Not ready yet ! */
 
 
-// Target advanced compression API
-// Not ready yet !!!
 
 typedef enum {
-    ZSTD_e_continue,  /* continue sending data, encoder transparently decides when to output result, depending on optimal conditions */
-    ZSTD_e_flush,     /* flush any data provided, compressed and buffered so far - frame will continue, future data can still reference previous data for better compression */
-    ZSTD_e_end        /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
+    ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
+    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
+    ZSTD_e_end         /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
 } ZSTD_EndDirective;
 
 /*! ZSTD_compressStream_generic() :
@@ -753,19 +766,18 @@ typedef enum {
  *            It is necessary to fully flush internal buffers
  *            before changing compression parameters or start a new compression job.
  */
-// Not ready yet !!!
 ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
-                                          void* dst, size_t dstCapacity, size_t* dstPos,
-                                    const void* src, size_t srcSize, size_t* srcPos,
+                                          ZSTD_outBuffer* output,
+                                          ZSTD_inBuffer* input,
                                           ZSTD_EndDirective endOp);
 
 /*! ZSTD_CCtx_reset() :
  *  Return a CCtx to clean state.
  *  Useful after an error, or to interrupt an ongoing compression job and start a new one.
- *  It's allowed to change compression parameters after a reset.
+ *  It's possible to modify compression parameters after a reset.
  *  Any internal data not yet flushed is cancelled.
  */
-ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
 
 
 
diff --git a/programs/fileio.c b/programs/fileio.c
index a36abcaff..e7c58a969 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -78,9 +78,6 @@
 #define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
 
 #define FIO_FRAMEHEADERSIZE  5    /* as a define, because needed to allocated table on stack */
-#define FSE_CHECKSUM_SEED    0
-
-#define CACHELINE 64
 
 #define DICTSIZE_MAX (32 MB)   /* protection against large input (attack scenario) */
 
@@ -106,6 +103,31 @@ static clock_t g_time = 0;
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
 
+/*-*************************************
+*  Errors
+***************************************/
+#ifndef DEBUG
+#  define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...)                                              \
+{                                                                          \
+    DISPLAYLEVEL(1, "zstd: ");                                             \
+    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__);  \
+    DISPLAYLEVEL(1, "error %i : ", error);                                 \
+    DISPLAYLEVEL(1, __VA_ARGS__);                                          \
+    DISPLAYLEVEL(1, " \n");                                                \
+    exit(error);                                                           \
+}
+
+#define CHECK(f) {                                   \
+    size_t const err = f;                            \
+    if (ZSTD_isError(err)) {                         \
+        DEBUGOUTPUT("%s \n", #f);                    \
+        EXM_THROW(11, "%s", ZSTD_getErrorName(err)); \
+}   }
+
+
 /* ************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
 ***************************************************************/
@@ -145,7 +167,7 @@ static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
 void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; }
 static U32 g_overwrite = 0;
 void FIO_overwriteMode(void) { g_overwrite=1; }
-static U32 g_sparseFileSupport = 1;   /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
+static U32 g_sparseFileSupport = 1;   /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
 void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
 static U32 g_dictIDFlag = 1;
 void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
@@ -181,23 +203,6 @@ void FIO_setOverlapLog(unsigned overlapLog){
 }
 
 
-/*-*************************************
-*  Exceptions
-***************************************/
-#ifndef DEBUG
-#  define DEBUG 0
-#endif
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...)                                             \
-{                                                                         \
-    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
-    DISPLAYLEVEL(1, "Error %i : ", error);                                \
-    DISPLAYLEVEL(1, __VA_ARGS__);                                         \
-    DISPLAYLEVEL(1, " \n");                                               \
-    exit(error);                                                          \
-}
-
-
 /*-*************************************
 *  Functions
 ***************************************/
@@ -225,7 +230,7 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
         f = stdin;
         SET_BINARY_MODE(stdin);
     } else {
-        if (!UTIL_isRegFile(srcFileName)) {
+        if (!UTIL_isRegularFile(srcFileName)) {
             DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
                             srcFileName);
             return NULL;
@@ -306,13 +311,13 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
 
     DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
     fileHandle = fopen(fileName, "rb");
-    if (fileHandle==0) EXM_THROW(31, "zstd: %s: %s", fileName, strerror(errno));
+    if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
     fileSize = UTIL_getFileSize(fileName);
     if (fileSize > DICTSIZE_MAX)
         EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
                         fileName, DICTSIZE_MAX >> 20);   /* avoid extreme cases */
     *bufferPtr = malloc((size_t)fileSize);
-    if (*bufferPtr==NULL) EXM_THROW(34, "zstd: %s", strerror(errno));
+    if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
     { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
       if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
     fclose(fileHandle);
@@ -331,7 +336,7 @@ typedef struct {
     size_t srcBufferSize;
     void*  dstBuffer;
     size_t dstBufferSize;
-#ifdef ZSTD_MULTITHREAD
+#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
     ZSTDMT_CCtx* cctx;
 #else
     ZSTD_CStream* cctx;
@@ -339,15 +344,19 @@ typedef struct {
 } cRess_t;
 
 static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
-                                    U64 srcSize, int srcRegFile,
+                                    U64 srcSize, int srcIsRegularFile,
                                     ZSTD_compressionParameters* comprParams) {
     cRess_t ress;
     memset(&ress, 0, sizeof(ress));
 
-#ifdef ZSTD_MULTITHREAD
+#ifdef ZSTD_NEWAPI
+    ress.cctx = ZSTD_createCCtx();
+    if (ress.cctx == NULL)
+        EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
+#elif defined(ZSTD_MULTITHREAD)
     ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
     if (ress.cctx == NULL)
-        EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
+        EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx");
     if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
         /* use complete window for overlap */
         ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);
@@ -356,22 +365,43 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
 #else
     ress.cctx = ZSTD_createCStream();
     if (ress.cctx == NULL)
-        EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
+        EXM_THROW(30, "allocation error : can't create ZSTD_CStream");
 #endif
     ress.srcBufferSize = ZSTD_CStreamInSize();
     ress.srcBuffer = malloc(ress.srcBufferSize);
     ress.dstBufferSize = ZSTD_CStreamOutSize();
     ress.dstBuffer = malloc(ress.dstBufferSize);
     if (!ress.srcBuffer || !ress.dstBuffer)
-        EXM_THROW(31, "zstd: allocation error : not enough memory");
+        EXM_THROW(31, "allocation error : not enough memory");
 
     /* dictionary */
     {   void* dictBuffer;
         size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName);   /* works with dictFileName==NULL */
         if (dictFileName && (dictBuffer==NULL))
-            EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
+            EXM_THROW(32, "allocation error : can't create dictBuffer");
+//#define ZSTD_NEWAPI
+#ifdef ZSTD_NEWAPI
+        {
+            /* frame parameters */
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, srcIsRegularFile) );
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
+            CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );
+            /* compression parameters */
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
+            /* dictionary */
+            CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
+        }
+#elif defined(ZSTD_MULTITHREAD)
         {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
-            params.fParams.contentSizeFlag = srcRegFile;
+            params.fParams.contentSizeFlag = srcIsRegularFile;
             params.fParams.checksumFlag = g_checksumFlag;
             params.fParams.noDictIDFlag = !g_dictIDFlag;
             if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
@@ -381,15 +411,24 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
             if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
             if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
             if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
-#ifdef ZSTD_MULTITHREAD
-            {   size_t const errorCode = ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
-                if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
-                ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
+            CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
+            ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
+        }
 #else
-            {   size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
-                if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
+        {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
+            params.fParams.contentSizeFlag = srcIsRegularFile;
+            params.fParams.checksumFlag = g_checksumFlag;
+            params.fParams.noDictIDFlag = !g_dictIDFlag;
+            if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
+            if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
+            if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
+            if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
+            if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
+            if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
+            if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
+            CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
+        }
 #endif
-        }   }
         free(dictBuffer);
     }
 
@@ -400,7 +439,7 @@ static void FIO_freeCResources(cRess_t ress)
 {
     free(ress.srcBuffer);
     free(ress.dstBuffer);
-#ifdef ZSTD_MULTITHREAD
+#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
     ZSTDMT_freeCCtx(ress.cctx);
 #else
     ZSTD_freeCStream(ress.cctx);   /* never fails */
@@ -705,41 +744,40 @@ static int FIO_compressFilename_internal(cRess_t ress,
     }
 
     /* init */
-#ifdef ZSTD_MULTITHREAD
-    {   size_t const resetError = ZSTDMT_resetCStream(ress.cctx, fileSize);
+#ifdef ZSTD_NEWAPI
+    /* nothing, reset is implied */
+#elif defined(ZSTD_MULTITHREAD)
+    CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) );
 #else
-    {   size_t const resetError = ZSTD_resetCStream(ress.cctx, fileSize);
+    CHECK( ZSTD_resetCStream(ress.cctx, fileSize) );
 #endif
-        if (ZSTD_isError(resetError))
-            EXM_THROW(21, "Error initializing compression : %s",
-                            ZSTD_getErrorName(resetError));
-    }
 
     /* Main compression loop */
     while (1) {
         /* Fill input Buffer */
         size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
+        ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
         if (inSize==0) break;
         readsize += inSize;
 
-        {   ZSTD_inBuffer  inBuff = { ress.srcBuffer, inSize, 0 };
-            while (inBuff.pos != inBuff.size) {
-                ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
-#ifdef ZSTD_MULTITHREAD
-                size_t const result = ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff);
+        while (inBuff.pos != inBuff.size) {
+            ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
+#ifdef ZSTD_NEWAPI
+            CHECK( ZSTD_compress_generic(ress.cctx,
+                        &outBuff, &inBuff, ZSTD_e_continue) );
+#elif defined(ZSTD_MULTITHREAD)
+            CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) );
 #else
-                size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
+            CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) );
 #endif
-                if (ZSTD_isError(result))
-                    EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result));
 
-                /* Write compressed stream */
-                if (outBuff.pos) {
-                    size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
-                    if (sizeCheck!=outBuff.pos)
-                        EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
-                    compressedfilesize += outBuff.pos;
-        }   }   }
+            /* Write compressed stream */
+            if (outBuff.pos) {
+                size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
+                if (sizeCheck!=outBuff.pos)
+                    EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
+                compressedfilesize += outBuff.pos;
+        }   }
         if (g_nbThreads > 1) {
             if (!fileSize)
                 DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20))
@@ -762,12 +800,18 @@ static int FIO_compressFilename_internal(cRess_t ress,
     {   size_t result = 1;
         while (result!=0) {   /* note : is there any possibility of endless loop ? */
             ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
-#ifdef ZSTD_MULTITHREAD
+#ifdef ZSTD_NEWAPI
+            ZSTD_inBuffer inBuff = { NULL, 0, 0};
+            result = ZSTD_compress_generic(ress.cctx,
+                        &outBuff, &inBuff, ZSTD_e_end);
+#elif defined(ZSTD_MULTITHREAD)
             result = ZSTDMT_endStream(ress.cctx, &outBuff);
 #else
             result = ZSTD_endStream(ress.cctx, &outBuff);
 #endif
-            if (ZSTD_isError(result)) EXM_THROW(26, "Compression error during frame end : %s", ZSTD_getErrorName(result));
+            if (ZSTD_isError(result))
+                EXM_THROW(26, "Compression error during frame end : %s",
+                            ZSTD_getErrorName(result));
             { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
               if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
             compressedfilesize += outBuff.pos;
@@ -856,9 +900,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
 {
     clock_t const start = clock();
     U64 const srcSize = UTIL_getFileSize(srcFileName);
-    int const regFile = UTIL_isRegFile(srcFileName);
+    int const isRegularFile = UTIL_isRegularFile(srcFileName);
 
-    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
+    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
     int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
 
     double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
@@ -879,8 +923,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
     char*  dstFileName = (char*)malloc(FNSPACE);
     size_t const suffixSize = suffix ? strlen(suffix) : 0;
     U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
-    int const regFile = (nbFiles != 1) ? 0 : UTIL_isRegFile(inFileNamesTable[0]);
-    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams);
+    int const isRegularFile = (nbFiles != 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]);
+    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
 
     /* init */
     if (dstFileName==NULL)
@@ -958,9 +1002,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
     /* dictionary */
     {   void* dictBuffer;
         size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName);
-        size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize);
-        if (ZSTD_isError(initError))
-            EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError));
+        CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) );
         free(dictBuffer);
     }
 
@@ -969,10 +1011,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
 
 static void FIO_freeDResources(dRess_t ress)
 {
-    size_t const errorCode = ZSTD_freeDStream(ress.dctx);
-    if (ZSTD_isError(errorCode))
-        EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s",
-                        ZSTD_getErrorName(errorCode));
+    CHECK( ZSTD_freeDStream(ress.dctx) );
     free(ress.srcBuffer);
     free(ress.dstBuffer);
 }
diff --git a/programs/util.h b/programs/util.h
index 54ae16227..dd971e0f8 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -208,7 +208,7 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
 }
 
 
-UTIL_STATIC int UTIL_isRegFile(const char* infilename)
+UTIL_STATIC int UTIL_isRegularFile(const char* infilename)
 {
     stat_t statbuf;
     return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */

From 009d604e0035bbfadd189ad9d8b883bbf83615cb Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 19 May 2017 10:17:59 -0700
Subject: [PATCH 010/109] ZSTD_compress_generic() supports multiple successive
 frames

also : clarified streaming API implementation
---
 lib/compress/zstd_compress.c | 245 ++++++++++++++++++-----------------
 lib/zstd.h                   |   6 +-
 tests/Makefile               |  16 +--
 tests/zstreamtest.c          |   9 +-
 4 files changed, 141 insertions(+), 135 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1d758dab7..b8fc8d987 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -41,7 +41,12 @@
 #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
 #  include 
    static unsigned g_debugLevel = ZSTD_DEBUG;
-#  define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); }
+#  define DEBUGLOG(l, ...) {                          \
+                if (l<=g_debugLevel) {                \
+                    fprintf(stderr, __FILE__ ": ");   \
+                    fprintf(stderr, __VA_ARGS__);     \
+                    fprintf(stderr, " \n");           \
+            }   }
 #else
 #  define DEBUGLOG(l, ...)      {}    /* disabled */
 #endif
@@ -185,6 +190,11 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
            + cctx->outBuffSize + cctx->inBuffSize;
 }
 
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
+{
+    return ZSTD_sizeof_CCtx(zcs);  /* same object */
+}
+
 /* private API call, for dictBuilder only */
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
 
@@ -423,12 +433,16 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
 }
 
 
-static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2)
+static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
+                                 ZSTD_compressionParameters cParams2)
 {
-    return (param1.cParams.hashLog  == param2.cParams.hashLog)
-         & (param1.cParams.chainLog == param2.cParams.chainLog)
-         & (param1.cParams.strategy == param2.cParams.strategy)   /* opt parser space */
-         & ((param1.cParams.searchLength==3) == (param2.cParams.searchLength==3));  /* hashlog3 space */
+    U32 bslog1 = MIN(cParams1.windowLog, 17);
+    U32 bslog2 = MIN(cParams2.windowLog, 17);
+    return (bslog1 == bslog2)   /* same block size */
+         & (cParams1.hashLog  == cParams2.hashLog)
+         & (cParams1.chainLog == cParams2.chainLog)
+         & (cParams1.strategy == cParams2.strategy)   /* opt parser space */
+         & ((cParams1.searchLength==3) == (cParams2.searchLength==3));  /* hashlog3 space */
 }
 
 /*! ZSTD_continueCCtx() :
@@ -459,11 +473,12 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
                                        ZSTD_parameters params, U64 frameContentSize,
                                        ZSTD_compResetPolicy_e const crp)
 {
-    DEBUGLOG(5, "ZSTD_resetCCtx_internal \n");
+    DEBUGLOG(5, "ZSTD_resetCCtx_internal : wlog=%u / old=%u",
+                params.cParams.windowLog, zc->params.cParams.windowLog);
 
     if (crp == ZSTDcrp_continue)
-        if (ZSTD_equivalentParams(params, zc->params)) {
-            DEBUGLOG(5, "ZSTD_equivalentParams()==1 \n");
+        if (ZSTD_equivalentParams(params.cParams, zc->params.cParams)) {
+            DEBUGLOG(5, "ZSTD_equivalentParams()==1");
             zc->fseCTables_ready = 0;
             zc->hufCTable_repeatMode = HUF_repeat_none;
             return ZSTD_continueCCtx(zc, params, frameContentSize);
@@ -514,7 +529,7 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
         /* init params */
         zc->params = params;
         zc->blockSize = blockSize;
-        DEBUGLOG(5, "blockSize = %uK \n", (U32)blockSize>>10);
+        DEBUGLOG(5, "blockSize = %uK", (U32)blockSize>>10);
         zc->frameContentSize = frameContentSize;
         zc->consumedSrcSize = 0;
 
@@ -544,7 +559,7 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
 
         /* opt parser space */
         if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) {
-            DEBUGLOG(5, "reserving optimal parser space ");
+            DEBUGLOG(5, "reserving optimal parser space");
             assert(((size_t)ptr & 3) == 0);  /* ensure ptr is properly aligned */
             zc->seqStore.litFreq = (U32*)ptr;
             zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<stage!=ZSTDcs_init) return ERROR(stage_wrong);
 
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
     {   ZSTD_parameters params = srcCCtx->params;
         params.fParams = fParams;
-        DEBUGLOG(5, "ZSTD_resetCCtx_internal : dictIDFlag : %u \n", !fParams.noDictIDFlag);
+        DEBUGLOG(5, "ZSTD_resetCCtx_internal : dictIDFlag : %u", !fParams.noDictIDFlag);
         ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
     }
 
@@ -1063,12 +1078,6 @@ _check_compressibility:
     return op - ostart;
 }
 
-#if 0 /* for debug */
-#  define STORESEQ_DEBUG
-#include    /* fprintf */
-U32 g_startDebug = 0;
-const BYTE* g_start = NULL;
-#endif
 
 /*! ZSTD_storeSeq() :
     Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
@@ -1077,16 +1086,16 @@ const BYTE* g_start = NULL;
 */
 MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
 {
-#ifdef STORESEQ_DEBUG
-    if (g_startDebug) {
-        const U32 pos = (U32)((const BYTE*)literals - g_start);
-        if (g_start==NULL) g_start = (const BYTE*)literals;
-        if ((pos > 1895000) && (pos < 1895300))
-            DEBUGLOG(5, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
-                   pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
-    }
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6)
+    static const BYTE* g_start = NULL;
+    U32 const pos = (U32)((const BYTE*)literals - g_start);
+    if (g_start==NULL) g_start = (const BYTE*)literals;
+    if ((pos > 0) && (pos < 1000000000))
+        DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u",
+               pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
 #endif
     /* copy Literals */
+    assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB);
     ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
     seqStorePtr->lit += litLength;
 
@@ -2713,7 +2722,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     size_t pos;
 
     if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
-    DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u \n",
+    DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
                 !params.fParams.noDictIDFlag, dictID,  dictIDSizeCode);
 
     MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
@@ -3024,7 +3033,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
     BYTE* op = ostart;
     size_t fhSize = 0;
 
-    DEBUGLOG(5, "ZSTD_writeEpilogue \n");
+    DEBUGLOG(5, "ZSTD_writeEpilogue");
     if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong);  /* init missing */
 
     /* special case : empty frame */
@@ -3062,8 +3071,9 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
                    const void* src, size_t srcSize)
 {
     size_t endResult;
-    size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize,
-                               1 /* frame mode */, 1 /* last chunk */);
+    size_t const cSize = ZSTD_compressContinue_internal(cctx,
+                                dst, dstCapacity, src, srcSize,
+                                1 /* frame mode */, 1 /* last chunk */);
     if (ZSTD_isError(cSize)) return cSize;
     endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
     if (ZSTD_isError(endResult)) return endResult;
@@ -3229,7 +3239,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
 {
     if (cdict==NULL) return ERROR(dictionary_wrong);  /* does not support NULL cdict */
-    DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced : dictIDFlag == %u \n", !fParams.noDictIDFlag);
+    DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced : dictIDFlag == %u", !fParams.noDictIDFlag);
     if (cdict->dictContentSize)
         CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) )
     else {
@@ -3246,7 +3256,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
 size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
     ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
-    DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u \n", !fParams.noDictIDFlag);
+    DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
     return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
 }
 
@@ -3318,7 +3328,7 @@ size_t ZSTD_CStreamOutSize(void)
 
 static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
-    DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag);
+    DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u", !zcs->params.fParams.noDictIDFlag);
 
     if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs, zcs->cdict, params.fParams, pledgedSrcSize))
     else CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, params, pledgedSrcSize));
@@ -3332,17 +3342,6 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters para
     return 0;   /* ready to go */
 }
 
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
-{
-
-    ZSTD_parameters params = zcs->params;
-    params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
-    if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
-        params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
-    }
-    return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
-}
-
 /* ZSTD_initCStream_internal() :
  * params are supposed validated at this stage
  * and zcs->cdict is supposed to be correct */
@@ -3372,15 +3371,28 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
         zcs->outBuffSize = outBuffSize;
     }
 
-    DEBUGLOG(5, "ZSTD_initCStream_stage2 : dictIDFlag == %u \n", !params.fParams.noDictIDFlag);
     return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
 }
 
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
+{
+
+    ZSTD_parameters params = zcs->params;
+    params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+    if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
+        params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
+    }
+    return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
+}
+
 /* ZSTD_initCStream_usingCDict_advanced() :
  * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
-size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams)
-{
-    if (!cdict) return ERROR(dictionary_wrong);  /* cannot handle NULL cdict (does not know what to do) */
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+                                            const ZSTD_CDict* cdict,
+                                            unsigned long long pledgedSrcSize,
+                                            ZSTD_frameParameters fParams)
+{   /* cannot handle NULL cdict (does not know what to do) */
+    if (!cdict) return ERROR(dictionary_wrong);
     {   ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
         params.fParams = fParams;
         zcs->cdict = cdict;
@@ -3409,7 +3421,6 @@ static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
         zcs->cdict = zcs->cdictLocal;
     }
 
-    DEBUGLOG(5, "ZSTD_initCStream_internal : dictIDFlag == %u \n", !params.fParams.noDictIDFlag);
     return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
 }
 
@@ -3418,8 +3429,6 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     CHECK_F( ZSTD_checkCParams(params.cParams) );
-    DEBUGLOG(5, "ZSTD_initCStream_advanced : pledgedSrcSize == %u \n", (U32)pledgedSrcSize);
-    DEBUGLOG(5, "wlog %u \n", params.cParams.windowLog);
     return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize);
 }
 
@@ -3442,11 +3451,6 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
     return ZSTD_initCStream_internal(zcs, NULL, 0, params, 0);
 }
 
-size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
-{
-    return ZSTD_sizeof_CCtx(zcs);   /* same object */
-}
-
 /*======   Compression   ======*/
 
 MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
@@ -3459,7 +3463,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
 static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                               void* dst, size_t* dstCapacityPtr,
                         const void* src, size_t* srcSizePtr,
-                              ZSTD_EndDirective const flush)
+                              ZSTD_EndDirective const flushMode)
 {
     U32 someMoreWork = 1;
     const char* const istart = (const char*)src;
@@ -3469,9 +3473,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
     char* const oend = ostart + *dstCapacityPtr;
     char* op = ostart;
 
-    DEBUGLOG(5, "ZSTD_compressStream_generic \n");
     assert(zcs->inBuff != NULL);
     assert(zcs->outBuff!= NULL);
+
     while (someMoreWork) {
         switch(zcs->streamStage)
         {
@@ -3481,56 +3485,79 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
             /* complete inBuffer */
             {   size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
                 size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip);
-                DEBUGLOG(5, "loading %u/%u \n", (U32)loaded, (U32)toLoad);
                 zcs->inBuffPos += loaded;
                 ip += loaded;
-                if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) {
-                    someMoreWork = 0; break;  /* not enough input to get a full block : stop there, wait for more */
-            }   }
+                if ( (flushMode == ZSTD_e_continue)
+                  && (zcs->inBuffPos < zcs->inBuffTarget) ) {
+                    /* not enough input to fill full block : stop here */
+                    someMoreWork = 0; break;
+                }
+                if ( (flushMode == ZSTD_e_flush)
+                  && (zcs->inBuffPos == zcs->inToCompress) ) {
+                    /* empty */
+                    someMoreWork = 0; break;
+                 }
+             }
             /* compress current block (note : this stage cannot be stopped in the middle) */
-            DEBUGLOG(5, "stream compression stage (flush==%u)\n", flush);
+            DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
             {   void* cDst;
                 size_t cSize;
                 size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
                 size_t oSize = oend-op;
+                unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
                 if (oSize >= ZSTD_compressBound(iSize))
-                    cDst = op;   /* compress directly into output buffer (avoid flush stage) */
+                    cDst = op;   /* compress directly into output buffer (skip flush stage) */
                 else
                     cDst = zcs->outBuff, oSize = zcs->outBuffSize;
-                cSize = (flush == ZSTD_e_end) ?
-                        ZSTD_compressEnd(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
-                        ZSTD_compressContinue(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
+                cSize = lastBlock ?
+                        ZSTD_compressEnd(zcs, cDst, oSize,
+                                    zcs->inBuff + zcs->inToCompress, iSize) :
+                        ZSTD_compressContinue(zcs, cDst, oSize,
+                                    zcs->inBuff + zcs->inToCompress, iSize);
                 if (ZSTD_isError(cSize)) return cSize;
-                DEBUGLOG(5, "cSize = %u \n", (U32)cSize);
-                if (flush == ZSTD_e_end) zcs->frameEnded = 1;
+                DEBUGLOG(5, "cSize = %u   (lastBlock:%u)", (U32)cSize, lastBlock);
+                zcs->frameEnded = lastBlock;
                 /* prepare next block */
                 zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
                 if (zcs->inBuffTarget > zcs->inBuffSize)
-                    zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;   /* note : inBuffTarget == blockSize <= inBuffSize */
+                    zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;
                 assert(zcs->inBuffTarget <= zcs->inBuffSize);
                 zcs->inToCompress = zcs->inBuffPos;
-                if (cDst == op) { op += cSize; break; }   /* no need to flush */
+                if (cDst == op) {  /* no need to flush */
+                    op += cSize;
+                    if (zcs->frameEnded) {
+                        DEBUGLOG(5, "Frame directly completed");
+                        someMoreWork = 0;
+                        zcs->streamStage = zcss_init;
+                    }
+                    break;
+                }
                 zcs->outBuffContentSize = cSize;
                 zcs->outBuffFlushedSize = 0;
-                zcs->streamStage = zcss_flush;   /* pass-through to flush stage */
+                zcs->streamStage = zcss_flush; /* pass-through to flush stage */
             }
 	    /* fall-through */
         case zcss_flush:
-            DEBUGLOG(5, "flush stage \n");
+            DEBUGLOG(5, "flush stage");
             {   size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
                 size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
-                DEBUGLOG(5, "toFlush: %u  ; flushed: %u \n", (U32)toFlush, (U32)flushed);
+                DEBUGLOG(5, "toFlush: %u  ; flushed: %u", (U32)toFlush, (U32)flushed);
                 op += flushed;
                 zcs->outBuffFlushedSize += flushed;
                 if (toFlush!=flushed) { someMoreWork = 0; break; }  /* dst too small to store flushed data : stop there */
                 zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
+                if (zcs->frameEnded) {
+                    DEBUGLOG(5, "Frame completed");
+                    someMoreWork = 0;
+                    zcs->streamStage = zcss_init;
+                    break;
+                }
                 zcs->streamStage = zcss_load;
                 break;
             }
 
         case zcss_final:
-            someMoreWork = 0;   /* do nothing */
-            break;
+            someMoreWork = 0; break;  /* useless */
 
         default:
             return ERROR(GENERIC);   /* impossible */
@@ -3573,10 +3600,10 @@ size_t ZSTD_compress_generic_integral (
     if (cctx->streamStage == zcss_init) {
         /* transparent reset */
         ZSTD_parameters params = cctx->params;
+        DEBUGLOG(5, "ZSTD_compress_generic_integral : transparent reset");
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
                                     cctx->frameContentSize, 0 /* dictSize */);
-        DEBUGLOG(5, "starting ZSTD_resetCStream");
         CHECK_F( ZSTD_initCStream_stage2(cctx, params, cctx->frameContentSize) );
     }
 
@@ -3626,46 +3653,26 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
     BYTE* const ostart = (BYTE*)(output->dst) + output->pos;
-    BYTE* const oend = (BYTE*)(output->dst) + output->size;
-    BYTE* op = ostart;
+    size_t srcSize = 0;
+    size_t sizeWritten = output->size - output->pos;
 
-    DEBUGLOG(5, "ZSTD_endStream (dstCapacity : %u) \n", (U32)(oend-op));
-    if (zcs->streamStage != zcss_final) {
-        /* flush whatever remains */
-        size_t srcSize = 0;
-        size_t sizeWritten = output->size - output->pos;
-        size_t const notEnded = ZSTD_compressStream_generic(zcs,
-                                    ostart, &sizeWritten,
-                                    &srcSize /* valid address */, &srcSize,
-                                    ZSTD_e_end);
-        size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
-        op += sizeWritten;
-        if (remainingToFlush) {
-            output->pos += sizeWritten;
-            return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */
-                   + ((zcs->params.fParams.checksumFlag > 0) * 4) /* optional 32-bits checksum */;
-        }
-        /* create epilogue */
-        zcs->streamStage = zcss_final;
-        zcs->outBuffContentSize = !notEnded ? 0 :
-            /* write epilogue, including final empty block, into outBuff */
-            ZSTD_compressEnd(zcs, zcs->outBuff, zcs->outBuffSize, NULL, 0);
-        if (ZSTD_isError(zcs->outBuffContentSize)) return zcs->outBuffContentSize;
-    }
+    DEBUGLOG(5, "calling ZSTD_endStream with outBuff=%u bytes",
+                (unsigned)sizeWritten);
 
-    /* flush epilogue */
-    {   size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
-        size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
-        op += flushed;
-        zcs->outBuffFlushedSize += flushed;
-        output->pos += op-ostart;
-        if (toFlush==flushed) zcs->streamStage = zcss_init;  /* end reached */
-        return toFlush - flushed;
-    }
+    size_t const result = ZSTD_compressStream_generic(zcs,
+                                ostart, &sizeWritten,
+                                &srcSize /* valid address */, &srcSize,
+                                ZSTD_e_end);
+    output->pos += sizeWritten;
+    if (ZSTD_isError(result)) return result;
+
+    DEBUGLOG(5, "remaining to flush : %u",
+            (unsigned)(zcs->outBuffContentSize - zcs->outBuffFlushedSize));
+
+    return zcs->outBuffContentSize - zcs->outBuffFlushedSize;
 }
 
 
-
 /*-=====  Pre-defined compression levels  =====-*/
 
 #define ZSTD_MAX_CLEVEL     22
@@ -3781,11 +3788,11 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
 /*! ZSTD_getCParams() :
 *   @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
 *   Size values are optional, provide 0 if not known or unused */
-ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
 {
     ZSTD_compressionParameters cp;
-    size_t const addedSize = srcSize ? 0 : 500;
-    U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1;
+    size_t const addedSize = srcSizeHint ? 0 : 500;
+    U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1;
     U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);   /* intentional underflow for srcSizeHint == 0 */
     if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* 0 == default; no negative compressionLevel yet */
     if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
@@ -3795,16 +3802,16 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
         if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
         if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
     }
-    cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
+    cp = ZSTD_adjustCParams(cp, srcSizeHint, dictSize);
     return cp;
 }
 
 /*! ZSTD_getParams() :
 *   same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`).
 *   All fields of `ZSTD_frameParameters` are set to default (0) */
-ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) {
+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) {
     ZSTD_parameters params;
-    ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
+    ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize);
     memset(¶ms, 0, sizeof(params));
     params.cParams = cParams;
     return params;
diff --git a/lib/zstd.h b/lib/zstd.h
index 338091834..7931bd5de 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -279,9 +279,11 @@ typedef struct ZSTD_outBuffer_s {
 *  ZSTD_endStream() instructs to finish a frame.
 *  It will perform a flush and write frame epilogue.
 *  The epilogue is required for decoders to consider a frame completed.
-*  Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
+*  ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
 *  In which case, call again ZSTD_endStream() to complete the flush.
-*  @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
+*  @return : 0 if frame fully completed and fully flushed,
+             or >0 if some data is still present within internal buffer
+                  (value is minimum size estimation for remaining data to flush, but it could be more)
 *            or an error code, which can be tested using ZSTD_isError().
 *
 * *******************************************************************/
diff --git a/tests/Makefile b/tests/Makefile
index c275c081f..0ee182fa8 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -26,16 +26,16 @@ PYTHON ?= python3
 TESTARTEFACT := versionsTest namespaceTest
 
 
-DEBUGFLAGS=-g -DZSTD_DEBUG=1
-CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-           -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
-           $(DEBUGFLAG)
-CFLAGS  ?= -O3
-CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+DEBUGFLAGS= -g -DZSTD_DEBUG=1
+CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+            -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
+CFLAGS   ?= -O3
+CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
             -Wstrict-prototypes -Wundef -Wformat-security \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls
+            -Wredundant-decls \
+            $(DEBUGFLAGS)
 CFLAGS  += $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
@@ -159,7 +159,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
 
-paramgrill : DEBUGFLAG =
+paramgrill : DEBUGFLAGS =
 paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
 	$(CC)      $(FLAGS) $^ -lm -o $@$(EXT)
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 6c4900ebb..e9e17031c 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -189,7 +189,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
 
     /* Basic compression test */
     DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
-    ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1);
+    { size_t const r = ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1);
+      if (ZSTD_isError(r)) goto _output_error; }
     outBuff.dst = (char*)(compressedBuffer)+cSize;
     outBuff.size = compressedBufferSize;
     outBuff.pos = 0;
@@ -797,13 +798,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
                 while (remainingToFlush) {
                     size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
                     size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
-                    U32 const enoughDstSize = (adjustedDstSize >= remainingToFlush);
                     outBuff.size = outBuff.pos + adjustedDstSize;
                     remainingToFlush = ZSTD_endStream(zc, &outBuff);
-                    CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
-                    CHECK (enoughDstSize && remainingToFlush,
-                           "ZSTD_endStream() not fully flushed (%u remaining), but enough space available (%u)",
-                           (U32)remainingToFlush, (U32)adjustedDstSize);
+                    CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
             }   }
             crcOrig = XXH64_digest(&xxhState);
             cSize = outBuff.pos;

From fa3671eac7097ce4ae9a1a4c53d2d1486c29d48f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 19 May 2017 10:51:30 -0700
Subject: [PATCH 011/109] changed ZSTD_BLOCKSIZE_ABSOLUTEMAX into
 ZSTD_BLOCKSIZE_MAX

Also :
change ZSTD_getBlockSizeMax() into ZSTD_getBlockSize()
created ZSTD_BLOCKSIZELOG_MAX
---
 doc/zstd_manual.html             | 10 ++++++----
 lib/compress/zstd_compress.c     | 26 +++++++++++++++-----------
 lib/decompress/zstd_decompress.c | 18 +++++++++---------
 lib/dictBuilder/zdict.c          |  8 ++++----
 lib/zstd.h                       |  7 ++++---
 tests/fuzzer.c                   |  1 +
 6 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index ff1ecd92f..de0ee4339 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -240,9 +240,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
   ZSTD_endStream() instructs to finish a frame.
   It will perform a flush and write frame epilogue.
   The epilogue is required for decoders to consider a frame completed.
-  Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
+  ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
   In which case, call again ZSTD_endStream() to complete the flush.
-  @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
+  @return : 0 if frame fully completed and fully flushed,
+             or >0 if some data is still present within internal buffer
+                  (value is minimum size estimation for remaining data to flush, but it could be more)
             or an error code, which can be tested using ZSTD_isError().
 
  
@@ -856,7 +858,7 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
       + compression : any ZSTD_compressBegin*() variant, including with dictionary
       + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
       + copyCCtx() and copyDCtx() can be used too
-    - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
+    - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
       + If input is larger than a block size, it's necessary to split input data into multiple blocks
       + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
         Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@@ -869,7 +871,7 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
         Use ZSTD_insertBlock() for such a case.
 

 
-
Raw zstd block functions
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+
Raw zstd block functions
size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
 size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert block into `dctx` history. Useful for uncompressed blocks */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b8fc8d987..f2d40edc9 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -410,7 +410,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
 
 size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
 {
-    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
+    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
     U32    const divider = (cParams.searchLength==3) ? 3 : 4;
     size_t const maxNbSeq = blockSize / divider;
     size_t const tokenSpace = blockSize + 11*maxNbSeq;
@@ -436,8 +436,8 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
 static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
                                  ZSTD_compressionParameters cParams2)
 {
-    U32 bslog1 = MIN(cParams1.windowLog, 17);
-    U32 bslog2 = MIN(cParams2.windowLog, 17);
+    U32 bslog1 = MIN(cParams1.windowLog, ZSTD_BLOCKSIZELOG_MAX);
+    U32 bslog2 = MIN(cParams2.windowLog, ZSTD_BLOCKSIZELOG_MAX);
     return (bslog1 == bslog2)   /* same block size */
          & (cParams1.hashLog  == cParams2.hashLog)
          & (cParams1.chainLog == cParams2.chainLog)
@@ -484,7 +484,7 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
             return ZSTD_continueCCtx(zc, params, frameContentSize);
         }
 
-    {   size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
+    {   size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
         U32    const divider = (params.cParams.searchLength==3) ? 3 : 4;
         size_t const maxNbSeq = blockSize / divider;
         size_t const tokenSpace = blockSize + 11*maxNbSeq;
@@ -2807,14 +2807,18 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
 }
 
 
-size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx)
+size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 {
-    return MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog);
+    U32 const cLevel = cctx->compressionLevel;
+    ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ?
+                                        cctx->params.cParams :
+                                        ZSTD_getCParams(cLevel, 0, 0);
+    return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
 }
 
 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
+    size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
     if (srcSize > blockSizeMax) return ERROR(srcSize_wrong);
     return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
 }
@@ -3308,7 +3312,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
 size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
 {
     size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
-    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
+    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
     size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
     size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
     size_t const streamingSize = inBuffSize + outBuffSize;
@@ -3319,11 +3323,11 @@ size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
 
 /*======   Initialization   ======*/
 
-size_t ZSTD_CStreamInSize(void)  { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
+size_t ZSTD_CStreamInSize(void)  { return ZSTD_BLOCKSIZE_MAX; }
 
 size_t ZSTD_CStreamOutSize(void)
 {
-    return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
+    return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
 }
 
 static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize)
@@ -3350,7 +3354,7 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
                                 unsigned long long pledgedSrcSize)
 {
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
-    zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
+    zcs->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
 
     /* allocate buffers */
     {   size_t const neededInBuffSize = ((size_t)1 << params.cParams.windowLog) + zcs->blockSize;
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 0c61960ec..03f8a9204 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -117,7 +117,7 @@ struct ZSTD_DCtx_s
     ZSTD_customMem customMem;
     size_t litSize;
     size_t rleSize;
-    BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
+    BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
 };  /* typedef'd to ZSTD_DCtx within "zstd.h" */
 
@@ -173,7 +173,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
 
 void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 {
-    size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
+    size_t const workSpaceSize = (ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
     memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize);  /* no need to copy workspace */
 }
 
@@ -483,7 +483,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                     litCSize = (lhc >> 22) + (istart[4] << 10);
                     break;
                 }
-                if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
+                if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
                 if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
 
                 if (HUF_isError((litEncType==set_repeat) ?
@@ -555,7 +555,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                     if (srcSize<4) return ERROR(corruption_detected);   /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
                     break;
                 }
-                if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
+                if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
                 memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH);
                 dctx->litPtr = dctx->litBuffer;
                 dctx->litSize = litSize;
@@ -1288,7 +1288,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 {   /* blockType == blockCompressed */
     const BYTE* ip = (const BYTE*)src;
 
-    if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong);
+    if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
 
     /* Decode literals section */
     {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
@@ -2120,8 +2120,8 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
 
 /* *** Initialization *** */
 
-size_t ZSTD_DStreamInSize(void)  { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
-size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
+size_t ZSTD_DStreamInSize(void)  { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
+size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
 
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
 {
@@ -2185,7 +2185,7 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
 size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader)
 {
     size_t const windowSize = fHeader.windowSize;
-    size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+    size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
     size_t const inBuffSize = blockSize;  /* no block can be larger */
     size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
     return sizeof(ZSTD_DStream) + ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
@@ -2281,7 +2281,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
 
             /* Adapt buffer sizes to frame header instructions */
-            {   size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+            {   size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
                 size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
                 zds->blockSize = blockSize;
                 if (zds->inBuffSize < blockSize) {
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 179e02eff..943ddde0f 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -576,7 +576,7 @@ typedef struct
 {
     ZSTD_CCtx* ref;
     ZSTD_CCtx* zc;
-    void* workPlace;   /* must be ZSTD_BLOCKSIZE_ABSOLUTEMAX allocated */
+    void* workPlace;   /* must be ZSTD_BLOCKSIZE_MAX allocated */
 } EStats_ress_t;
 
 #define MAXREPOFFSET 1024
@@ -585,14 +585,14 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
                             U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
                             const void* src, size_t srcSize, U32 notificationLevel)
 {
-    size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog);
+    size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog);
     size_t cSize;
 
     if (srcSize > blockSizeMax) srcSize = blockSizeMax;   /* protection vs large samples */
     {  size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0);
             if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
     }
-    cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);
+    cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
     if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
 
     if (cSize) {  /* if == 0; block is not compressible */
@@ -700,7 +700,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     /* init */
     esr.ref = ZSTD_createCCtx();
     esr.zc = ZSTD_createCCtx();
-    esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+    esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
     if (!esr.ref || !esr.zc || !esr.workPlace) {
         eSize = ERROR(memory_allocation);
         DISPLAYLEVEL(1, "Not enough memory \n");
diff --git a/lib/zstd.h b/lib/zstd.h
index 7931bd5de..5e4bc156e 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -1003,7 +1003,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
       + compression : any ZSTD_compressBegin*() variant, including with dictionary
       + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
       + copyCCtx() and copyDCtx() can be used too
-    - Block size is limited, it must be <= ZSTD_getBlockSizeMax() <= ZSTD_BLOCKSIZE_ABSOLUTEMAX
+    - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX
       + If input is larger than a block size, it's necessary to split input data into multiple blocks
       + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead.
         Frame metadata is not that costly, and quickly becomes negligible as source size grows larger.
@@ -1016,9 +1016,10 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
         Use ZSTD_insertBlock() for such a case.
 */
 
-#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024)   /* define, for static allocation */
+#define ZSTD_BLOCKSIZELOG_MAX 17
+#define ZSTD_BLOCKSIZE_MAX   (1<= blockSize);
         cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
         if (ZSTD_isError(cSize)) goto _output_error;
         DISPLAYLEVEL(4, "OK \n");

From 48855fa0d2308217d99c86cfc9d83818260e4dc4 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 19 May 2017 10:56:11 -0700
Subject: [PATCH 012/109] fixed declaration-after-statement warning

---
 lib/compress/zstd_compress.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index f2d40edc9..69b42cc80 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3660,9 +3660,6 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
     size_t srcSize = 0;
     size_t sizeWritten = output->size - output->pos;
 
-    DEBUGLOG(5, "calling ZSTD_endStream with outBuff=%u bytes",
-                (unsigned)sizeWritten);
-
     size_t const result = ZSTD_compressStream_generic(zcs,
                                 ostart, &sizeWritten,
                                 &srcSize /* valid address */, &srcSize,
@@ -3670,7 +3667,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
     output->pos += sizeWritten;
     if (ZSTD_isError(result)) return result;
 
-    DEBUGLOG(5, "remaining to flush : %u",
+    DEBUGLOG(5, "ZSTD_endStream : remaining to flush : %u",
             (unsigned)(zcs->outBuffContentSize - zcs->outBuffFlushedSize));
 
     return zcs->outBuffContentSize - zcs->outBuffFlushedSize;

From 334a288d0dda5cc43a80311dabde5b990c685ca6 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 19 May 2017 11:04:41 -0700
Subject: [PATCH 013/109] ZSTD_CCtx_setParameter() only works during
 initialization stage

and generate a stage_wrong error otherwise.
---
 lib/compress/zstd_compress.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 69b42cc80..a3d4ebe7f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -223,7 +223,14 @@ static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
 
 size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
 {
-#   define CLAMPCHECK(val,min,max) { if ((valmax)) return ERROR(compressionParameter_unsupported); }
+#   define CLAMPCHECK(val,min,max) {                         \
+        if ((valmax)) {                         \
+            return ERROR(compressionParameter_unsupported);  \
+    }   }
+
+    if (cctx->streamStage != zcss_init) {
+        return ERROR(stage_wrong);
+    }
 
     switch(param)
     {

From 60a557e7fd597142e9221b50c90c6fe4af7776e8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 19 May 2017 11:27:43 -0700
Subject: [PATCH 014/109] fixing symbol.c test

I believe it would be better to rely on fuzzer tests compiled with dll
---
 tests/symbols.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/symbols.c b/tests/symbols.c
index ade3aa02c..5139a6548 100644
--- a/tests/symbols.c
+++ b/tests/symbols.c
@@ -95,7 +95,7 @@ static const void *symbols[] = {
   &ZSTD_nextSrcSizeToDecompress,
   &ZSTD_decompressContinue,
   &ZSTD_nextInputType,
-  &ZSTD_getBlockSizeMax,
+  &ZSTD_getBlockSize,
   &ZSTD_compressBlock,
   &ZSTD_decompressBlock,
   &ZSTD_insertBlock,

From 8b21ec42a93ce2154eb50afd69e612b4bea764c9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 19 May 2017 19:46:15 -0700
Subject: [PATCH 015/109] ZSTD_compress_generic() can handle dictionary
 compression

---
 lib/compress/zstd_compress.c | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index a3d4ebe7f..94f006257 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -332,15 +332,21 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
 
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
 {
+    DEBUGLOG(5, "ZSTD_CCtx_loadDictionary : dictSize = %u",
+                (unsigned)dictSize);
     ZSTD_freeCDict(cctx->cdictLocal);  /* in case one already exists */
     if (dict==NULL || dictSize==0) {   /* no dictionary mode */
         cctx->cdictLocal = NULL;
         cctx->cdict = NULL;
     } else {
+        ZSTD_compressionParameters const cParams =
+                cctx->compressionLevel == ZSTD_CLEVEL_CUSTOM ?
+                cctx->params.cParams :
+                ZSTD_getCParams(cctx->compressionLevel, 0, dictSize);
         cctx->cdictLocal = ZSTD_createCDict_advanced(
                                 dict, dictSize,
                                 0 /* byReference */,
-                                cctx->params.cParams, cctx->customMem);
+                                cParams, cctx->customMem);
         cctx->cdict = cctx->cdictLocal;
         if (cctx->cdictLocal == NULL)
             return ERROR(memory_allocation);
@@ -3175,6 +3181,7 @@ static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_
 ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
                                       ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
 {
+    DEBUGLOG(5, "ZSTD_createCDict_advanced");
     if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
     if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
@@ -3182,6 +3189,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
         ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
 
         if (!cdict || !cctx) {
+            DEBUGLOG(5, "!cdict || !cctx");
             ZSTD_free(cdict, customMem);
             ZSTD_freeCCtx(cctx);
             return NULL;
@@ -3192,16 +3200,25 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
             cdict->dictContent = dictBuffer;
         } else {
             void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
-            if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; }
+            if (!internalBuffer) {
+                DEBUGLOG(5, "!internalBuffer");
+                ZSTD_free(cctx, customMem);
+                ZSTD_free(cdict, customMem);
+                return NULL;
+            }
             memcpy(internalBuffer, dictBuffer, dictSize);
             cdict->dictBuffer = internalBuffer;
             cdict->dictContent = internalBuffer;
         }
 
-        {   ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksumFlag */, 0 /* noDictIDFlag */ };   /* dummy */
+        {   ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */,
+                    0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */
             ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams);
-            size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
+            size_t const errorCode = ZSTD_compressBegin_advanced(cctx,
+                                cdict->dictContent, dictSize, params, 0);
             if (ZSTD_isError(errorCode)) {
+                DEBUGLOG(5, "ZSTD_compressBegin_advanced error : %s",
+                            ZSTD_getErrorName(errorCode));
                 ZSTD_free(cdict->dictBuffer, customMem);
                 ZSTD_free(cdict, customMem);
                 ZSTD_freeCCtx(cctx);
@@ -3532,7 +3549,10 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
                 if (zcs->inBuffTarget > zcs->inBuffSize)
                     zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;
-                assert(zcs->inBuffTarget <= zcs->inBuffSize);
+                DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u",
+                         (U32)zcs->inBuffTarget, (U32)zcs->inBuffSize);
+                if (!lastBlock)
+                    assert(zcs->inBuffTarget <= zcs->inBuffSize);
                 zcs->inToCompress = zcs->inBuffPos;
                 if (cDst == op) {  /* no need to flush */
                     op += cSize;

From 24de7b0346c0bf867482af0c4d987164453d1970 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 22 May 2017 13:05:45 -0700
Subject: [PATCH 016/109] Implemented ZSTD_CCtx_refCDict()

---
 lib/compress/zstd_compress.c | 14 ++++++--------
 lib/zstd.h                   |  2 +-
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 94f006257..ae49938f7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -228,9 +228,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
             return ERROR(compressionParameter_unsupported);  \
     }   }
 
-    if (cctx->streamStage != zcss_init) {
-        return ERROR(stage_wrong);
-    }
+    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
 
     switch(param)
     {
@@ -326,14 +324,14 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
+    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     cctx->frameContentSize = pledgedSrcSize;
     return 0;
 }
 
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
 {
-    DEBUGLOG(5, "ZSTD_CCtx_loadDictionary : dictSize = %u",
-                (unsigned)dictSize);
+    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     ZSTD_freeCDict(cctx->cdictLocal);  /* in case one already exists */
     if (dict==NULL || dictSize==0) {   /* no dictionary mode */
         cctx->cdictLocal = NULL;
@@ -358,13 +356,14 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
 {
     (void)cctx; (void)prefix; (void)prefixSize; /* to be done later */
+    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     return ERROR(compressionParameter_unsupported);
 }
 
-/* Not ready yet ! */
 ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
-    (void)cctx; (void)cdict;  /* to be done later */
+    if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+    cctx->cdict = cdict;
     return ERROR(compressionParameter_unsupported);
 }
 
@@ -384,7 +383,6 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
     return 0;
 }
 
-
 /** ZSTD_cycleLog() :
  *  condition for correct operation : hashLog > 1 */
 static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
diff --git a/lib/zstd.h b/lib/zstd.h
index 5a0206a01..83fe9d30f 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -746,7 +746,7 @@ ZSTDLIB_API size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict
  *           Adding a new dictionary effectively "discards" any previous one.
  *  Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
  */
-ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);  /* Not ready yet ! */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 
 
 

From 1ad7c82eb50a0d502fa9616920c65c53a1a01c1a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 22 May 2017 17:06:04 -0700
Subject: [PATCH 017/109] Implemented separation between requested and applied
 parameters

first version to pass cli tests with -DZSTD_NEWAPI
---
 lib/compress/zstd_compress.c | 176 +++++++++++++++++++----------------
 lib/compress/zstd_opt.h      |  21 +++--
 2 files changed, 108 insertions(+), 89 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ae49938f7..0e694d2ef 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -110,7 +110,8 @@ struct ZSTD_CCtx_s {
     U32   repToConfirm[ZSTD_REP_NUM];
     U32   dictID;
     int   compressionLevel;
-    ZSTD_parameters params;
+    ZSTD_parameters requestedParams;
+    ZSTD_parameters appliedParams;
     void* workSpace;
     size_t workSpaceSize;
     size_t blockSize;
@@ -198,7 +199,7 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
 /* private API call, for dictBuilder only */
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
 
-static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx) { return cctx->params; }
+static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx) { return cctx->appliedParams; }
 
 /* older variant; will be deprecated */
 size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
@@ -215,9 +216,9 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
 #define ZSTD_CLEVEL_CUSTOM 999
 static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
 {
-    if (cctx->compressionLevel==ZSTD_CLEVEL_CUSTOM)  return;
-    cctx->params.cParams = ZSTD_getCParams(cctx->compressionLevel,
-                                           cctx->frameContentSize, 0);
+    if (cctx->compressionLevel==ZSTD_CLEVEL_CUSTOM) return;
+    cctx->requestedParams.cParams = ZSTD_getCParams(cctx->compressionLevel,
+                                                    cctx->frameContentSize, 0);
     cctx->compressionLevel = ZSTD_CLEVEL_CUSTOM;
 }
 
@@ -242,49 +243,49 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
             if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
             ZSTD_cLevelToCParams(cctx);
-            cctx->params.cParams.windowLog = value;
+            cctx->requestedParams.cParams.windowLog = value;
             return 0;
 
     case ZSTD_p_hashLog :
             if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
             ZSTD_cLevelToCParams(cctx);
-            cctx->params.cParams.hashLog = value;
+            cctx->requestedParams.cParams.hashLog = value;
             return 0;
 
     case ZSTD_p_chainLog :
             if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
             ZSTD_cLevelToCParams(cctx);
-            cctx->params.cParams.chainLog = value;
+            cctx->requestedParams.cParams.chainLog = value;
             return 0;
 
     case ZSTD_p_searchLog :
             if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
             ZSTD_cLevelToCParams(cctx);
-            cctx->params.cParams.searchLog = value;
+            cctx->requestedParams.cParams.searchLog = value;
             return 0;
 
     case ZSTD_p_minMatch :
             if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
             ZSTD_cLevelToCParams(cctx);
-            cctx->params.cParams.searchLength = value;
+            cctx->requestedParams.cParams.searchLength = value;
             return 0;
 
     case ZSTD_p_targetLength :
             if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
             ZSTD_cLevelToCParams(cctx);
-            cctx->params.cParams.targetLength = value;
+            cctx->requestedParams.cParams.targetLength = value;
             return 0;
 
     case ZSTD_p_compressionStrategy :
             if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
             CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
             ZSTD_cLevelToCParams(cctx);
-            cctx->params.cParams.strategy = (ZSTD_strategy)value;
+            cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value;
             return 0;
 
 #if 0
@@ -293,15 +294,16 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 #endif
 
     case ZSTD_p_contentSizeFlag : /* Content size will be written in frame header _when known_ (default:1) */
-            cctx->params.fParams.contentSizeFlag = value>0;
+            cctx->requestedParams.fParams.contentSizeFlag = value>0;
             return 0;
 
     case ZSTD_p_checksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
-            cctx->params.fParams.checksumFlag = value>0;
+            cctx->requestedParams.fParams.checksumFlag = value>0;
             return 0;
 
-    case ZSTD_p_dictIDFlag : /* When applicable, the dictID of used dictionary will be provided in frame header (default:1) */
-            cctx->params.fParams.noDictIDFlag = value==0;
+    case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
+            DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
+            cctx->requestedParams.fParams.noDictIDFlag = (value==0);
             return 0;
 
     case ZSTD_p_refDictContent :   /* to be done later */
@@ -339,7 +341,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
     } else {
         ZSTD_compressionParameters const cParams =
                 cctx->compressionLevel == ZSTD_CLEVEL_CUSTOM ?
-                cctx->params.cParams :
+                cctx->requestedParams.cParams :
                 ZSTD_getCParams(cctx->compressionLevel, 0, dictSize);
         cctx->cdictLocal = ZSTD_createCDict_advanced(
                                 dict, dictSize,
@@ -461,7 +463,7 @@ static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
 static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 frameContentSize)
 {
     U32 const end = (U32)(cctx->nextSrc - cctx->base);
-    cctx->params = params;
+    cctx->appliedParams = params;
     cctx->frameContentSize = frameContentSize;
     cctx->consumedSrcSize = 0;
     cctx->lowLimit = end;
@@ -485,10 +487,10 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
                                        ZSTD_compResetPolicy_e const crp)
 {
     DEBUGLOG(5, "ZSTD_resetCCtx_internal : wlog=%u / old=%u",
-                params.cParams.windowLog, zc->params.cParams.windowLog);
+                params.cParams.windowLog, zc->appliedParams.cParams.windowLog);
 
     if (crp == ZSTDcrp_continue)
-        if (ZSTD_equivalentParams(params.cParams, zc->params.cParams)) {
+        if (ZSTD_equivalentParams(params.cParams, zc->appliedParams.cParams)) {
             DEBUGLOG(5, "ZSTD_equivalentParams()==1");
             zc->fseCTables_ready = 0;
             zc->hufCTable_repeatMode = HUF_repeat_none;
@@ -538,7 +540,7 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
         }   }
 
         /* init params */
-        zc->params = params;
+        zc->appliedParams = params;
         zc->blockSize = blockSize;
         DEBUGLOG(5, "blockSize = %uK", (U32)blockSize>>10);
         zc->frameContentSize = frameContentSize;
@@ -618,22 +620,24 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
  *  Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
  *  pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1
  *  @return : 0, or an error code */
-size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx,
-                              ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize)
+static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
+                            const ZSTD_CCtx* srcCCtx,
+                            ZSTD_frameParameters fParams,
+                            unsigned long long pledgedSrcSize)
 {
     DEBUGLOG(5, "ZSTD_copyCCtx_internal");
     if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
 
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
-    {   ZSTD_parameters params = srcCCtx->params;
+    {   ZSTD_parameters params = srcCCtx->appliedParams;
         params.fParams = fParams;
         DEBUGLOG(5, "ZSTD_resetCCtx_internal : dictIDFlag : %u", !fParams.noDictIDFlag);
         ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
     }
 
     /* copy tables */
-    {   size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
-        size_t const hSize =  (size_t)1 << srcCCtx->params.cParams.hashLog;
+    {   size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->appliedParams.cParams.chainLog);
+        size_t const hSize =  (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
         size_t const h3Size = (size_t)1 << srcCCtx->hashLog3;
         size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
         assert((U32*)dstCCtx->chainTable == (U32*)dstCCtx->hashTable + hSize);  /* chainTable must follow hashTable */
@@ -696,10 +700,10 @@ static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reduce
 *   rescale all indexes to avoid future overflow (indexes are U32) */
 static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
 {
-    { U32 const hSize = 1 << zc->params.cParams.hashLog;
+    { U32 const hSize = 1 << zc->appliedParams.cParams.hashLog;
       ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); }
 
-    { U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog);
+    { U32 const chainSize = (zc->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->appliedParams.cParams.chainLog);
       ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); }
 
     { U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
@@ -795,7 +799,7 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
 
     if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall);   /* not enough space for compression */
     {   HUF_repeat repeat = zc->hufCTable_repeatMode;
-        int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
+        int const preferRepeat = zc->appliedParams.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
         cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11,
                                       zc->entropyScratchSpace, entropyScratchSpace_size, zc->hufCTable, &repeat, preferRepeat)
@@ -884,7 +888,7 @@ MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
                               void* dst, size_t dstCapacity,
                               size_t srcSize)
 {
-    const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
+    const int longOffsets = zc->appliedParams.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
     const seqStore_t* seqStorePtr = &(zc->seqStore);
     U32 count[MaxSeq+1];
     S16 norm[MaxSeq+1];
@@ -1280,7 +1284,7 @@ static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
 static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
 {
     U32* const hashTable = zc->hashTable;
-    U32  const hBits = zc->params.cParams.hashLog;
+    U32  const hBits = zc->appliedParams.cParams.hashLog;
     const BYTE* const base = zc->base;
     const BYTE* ip = base + zc->nextToUpdate;
     const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
@@ -1299,7 +1303,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
                                const U32 mls)
 {
     U32* const hashTable = cctx->hashTable;
-    U32  const hBits = cctx->params.cParams.hashLog;
+    U32  const hBits = cctx->appliedParams.cParams.hashLog;
     seqStore_t* seqStorePtr = &(cctx->seqStore);
     const BYTE* const base = cctx->base;
     const BYTE* const istart = (const BYTE*)src;
@@ -1384,7 +1388,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
 static void ZSTD_compressBlock_fast(ZSTD_CCtx* ctx,
                        const void* src, size_t srcSize)
 {
-    const U32 mls = ctx->params.cParams.searchLength;
+    const U32 mls = ctx->appliedParams.cParams.searchLength;
     switch(mls)
     {
     default: /* includes case 3 */
@@ -1405,7 +1409,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
                                  const U32 mls)
 {
     U32* hashTable = ctx->hashTable;
-    const U32 hBits = ctx->params.cParams.hashLog;
+    const U32 hBits = ctx->appliedParams.cParams.hashLog;
     seqStore_t* seqStorePtr = &(ctx->seqStore);
     const BYTE* const base = ctx->base;
     const BYTE* const dictBase = ctx->dictBase;
@@ -1498,7 +1502,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
 static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
                          const void* src, size_t srcSize)
 {
-    U32 const mls = ctx->params.cParams.searchLength;
+    U32 const mls = ctx->appliedParams.cParams.searchLength;
     switch(mls)
     {
     default: /* includes case 3 */
@@ -1520,9 +1524,9 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
 static void ZSTD_fillDoubleHashTable (ZSTD_CCtx* cctx, const void* end, const U32 mls)
 {
     U32* const hashLarge = cctx->hashTable;
-    U32  const hBitsL = cctx->params.cParams.hashLog;
+    U32  const hBitsL = cctx->appliedParams.cParams.hashLog;
     U32* const hashSmall = cctx->chainTable;
-    U32  const hBitsS = cctx->params.cParams.chainLog;
+    U32  const hBitsS = cctx->appliedParams.cParams.chainLog;
     const BYTE* const base = cctx->base;
     const BYTE* ip = base + cctx->nextToUpdate;
     const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
@@ -1542,9 +1546,9 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
                                  const U32 mls)
 {
     U32* const hashLong = cctx->hashTable;
-    const U32 hBitsL = cctx->params.cParams.hashLog;
+    const U32 hBitsL = cctx->appliedParams.cParams.hashLog;
     U32* const hashSmall = cctx->chainTable;
-    const U32 hBitsS = cctx->params.cParams.chainLog;
+    const U32 hBitsS = cctx->appliedParams.cParams.chainLog;
     seqStore_t* seqStorePtr = &(cctx->seqStore);
     const BYTE* const base = cctx->base;
     const BYTE* const istart = (const BYTE*)src;
@@ -1654,7 +1658,7 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
 
 static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
 {
-    const U32 mls = ctx->params.cParams.searchLength;
+    const U32 mls = ctx->appliedParams.cParams.searchLength;
     switch(mls)
     {
     default: /* includes case 3 */
@@ -1675,9 +1679,9 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
                                  const U32 mls)
 {
     U32* const hashLong = ctx->hashTable;
-    U32  const hBitsL = ctx->params.cParams.hashLog;
+    U32  const hBitsL = ctx->appliedParams.cParams.hashLog;
     U32* const hashSmall = ctx->chainTable;
-    U32  const hBitsS = ctx->params.cParams.chainLog;
+    U32  const hBitsS = ctx->appliedParams.cParams.chainLog;
     seqStore_t* seqStorePtr = &(ctx->seqStore);
     const BYTE* const base = ctx->base;
     const BYTE* const dictBase = ctx->dictBase;
@@ -1804,7 +1808,7 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
 static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx,
                          const void* src, size_t srcSize)
 {
-    U32 const mls = ctx->params.cParams.searchLength;
+    U32 const mls = ctx->appliedParams.cParams.searchLength;
     switch(mls)
     {
     default: /* includes case 3 */
@@ -1830,10 +1834,10 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
                           U32 extDict)
 {
     U32*   const hashTable = zc->hashTable;
-    U32    const hashLog = zc->params.cParams.hashLog;
+    U32    const hashLog = zc->appliedParams.cParams.hashLog;
     size_t const h  = ZSTD_hashPtr(ip, hashLog, mls);
     U32*   const bt = zc->chainTable;
-    U32    const btLog  = zc->params.cParams.chainLog - 1;
+    U32    const btLog  = zc->appliedParams.cParams.chainLog - 1;
     U32    const btMask = (1 << btLog) - 1;
     U32 matchIndex = hashTable[h];
     size_t commonLengthSmaller=0, commonLengthLarger=0;
@@ -1935,10 +1939,10 @@ static size_t ZSTD_insertBtAndFindBestMatch (
                         U32 extDict)
 {
     U32*   const hashTable = zc->hashTable;
-    U32    const hashLog = zc->params.cParams.hashLog;
+    U32    const hashLog = zc->appliedParams.cParams.hashLog;
     size_t const h  = ZSTD_hashPtr(ip, hashLog, mls);
     U32*   const bt = zc->chainTable;
-    U32    const btLog  = zc->params.cParams.chainLog - 1;
+    U32    const btLog  = zc->appliedParams.cParams.chainLog - 1;
     U32    const btMask = (1 << btLog) - 1;
     U32 matchIndex  = hashTable[h];
     size_t commonLengthSmaller=0, commonLengthLarger=0;
@@ -2098,9 +2102,9 @@ FORCE_INLINE
 U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
 {
     U32* const hashTable  = zc->hashTable;
-    const U32 hashLog = zc->params.cParams.hashLog;
+    const U32 hashLog = zc->appliedParams.cParams.hashLog;
     U32* const chainTable = zc->chainTable;
-    const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1;
+    const U32 chainMask = (1 << zc->appliedParams.cParams.chainLog) - 1;
     const BYTE* const base = zc->base;
     const U32 target = (U32)(ip - base);
     U32 idx = zc->nextToUpdate;
@@ -2126,7 +2130,7 @@ size_t ZSTD_HcFindBestMatch_generic (
                         const U32 maxNbAttempts, const U32 mls, const U32 extDict)
 {
     U32* const chainTable = zc->chainTable;
-    const U32 chainSize = (1 << zc->params.cParams.chainLog);
+    const U32 chainSize = (1 << zc->appliedParams.cParams.chainLog);
     const U32 chainMask = chainSize-1;
     const BYTE* const base = zc->base;
     const BYTE* const dictBase = zc->dictBase;
@@ -2220,8 +2224,8 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
     const BYTE* const ilimit = iend - 8;
     const BYTE* const base = ctx->base + ctx->dictLimit;
 
-    U32 const maxSearches = 1 << ctx->params.cParams.searchLog;
-    U32 const mls = ctx->params.cParams.searchLength;
+    U32 const maxSearches = 1 << ctx->appliedParams.cParams.searchLog;
+    U32 const mls = ctx->appliedParams.cParams.searchLength;
 
     typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit,
                         size_t* offsetPtr,
@@ -2384,8 +2388,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
     const BYTE* const dictEnd  = dictBase + dictLimit;
     const BYTE* const dictStart  = dictBase + ctx->lowLimit;
 
-    const U32 maxSearches = 1 << ctx->params.cParams.searchLog;
-    const U32 mls = ctx->params.cParams.searchLength;
+    const U32 maxSearches = 1 << ctx->appliedParams.cParams.searchLog;
+    const U32 mls = ctx->appliedParams.cParams.searchLength;
 
     typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit,
                         size_t* offsetPtr,
@@ -2625,7 +2629,7 @@ static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int
 
 static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-    ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit);
+    ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, zc->lowLimit < zc->dictLimit);
     const BYTE* const base = zc->base;
     const BYTE* const istart = (const BYTE*)src;
     const U32 current = (U32)(istart-base);
@@ -2655,9 +2659,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
     const BYTE* ip = (const BYTE*)src;
     BYTE* const ostart = (BYTE*)dst;
     BYTE* op = ostart;
-    U32 const maxDist = 1 << cctx->params.cParams.windowLog;
+    U32 const maxDist = 1 << cctx->appliedParams.cParams.windowLog;
 
-    if (cctx->params.fParams.checksumFlag && srcSize)
+    if (cctx->appliedParams.fParams.checksumFlag && srcSize)
         XXH64_update(&cctx->xxhState, src, srcSize);
 
     while (remaining) {
@@ -2670,9 +2674,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
 
         /* preemptive overflow correction */
         if (cctx->lowLimit > (3U<<29)) {
-            U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
+            U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->appliedParams.cParams.hashLog, cctx->appliedParams.cParams.strategy)) - 1;
             U32 const current = (U32)(ip - cctx->base);
-            U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog);
+            U32 const newCurrent = (current & cycleMask) + (1 << cctx->appliedParams.cParams.windowLog);
             U32 const correction = current - newCurrent;
             ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30);
             ZSTD_reduceIndex(cctx, correction);
@@ -2770,7 +2774,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
     if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong);   /* missing init (ZSTD_compressBegin) */
 
     if (frame && (cctx->stage==ZSTDcs_init)) {
-        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID);
+        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, cctx->frameContentSize, cctx->dictID);
         if (ZSTD_isError(fhSize)) return fhSize;
         dstCapacity -= fhSize;
         dst = (char*)dst + fhSize;
@@ -2822,7 +2826,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 {
     U32 const cLevel = cctx->compressionLevel;
     ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ?
-                                        cctx->params.cParams :
+                                        cctx->appliedParams.cParams :
                                         ZSTD_getCParams(cLevel, 0, 0);
     return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
 }
@@ -2853,28 +2857,28 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
     zc->nextSrc = iend;
     if (srcSize <= HASH_READ_SIZE) return 0;
 
-    switch(zc->params.cParams.strategy)
+    switch(zc->appliedParams.cParams.strategy)
     {
     case ZSTD_fast:
-        ZSTD_fillHashTable (zc, iend, zc->params.cParams.searchLength);
+        ZSTD_fillHashTable (zc, iend, zc->appliedParams.cParams.searchLength);
         break;
 
     case ZSTD_dfast:
-        ZSTD_fillDoubleHashTable (zc, iend, zc->params.cParams.searchLength);
+        ZSTD_fillDoubleHashTable (zc, iend, zc->appliedParams.cParams.searchLength);
         break;
 
     case ZSTD_greedy:
     case ZSTD_lazy:
     case ZSTD_lazy2:
         if (srcSize >= HASH_READ_SIZE)
-            ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->params.cParams.searchLength);
+            ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->appliedParams.cParams.searchLength);
         break;
 
     case ZSTD_btlazy2:
     case ZSTD_btopt:
     case ZSTD_btultra:
         if (srcSize >= HASH_READ_SIZE)
-            ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength);
+            ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
         break;
 
     default:
@@ -2918,7 +2922,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
     BYTE scratchBuffer[1<dictID = cctx->params.fParams.noDictIDFlag ? 0 :  MEM_readLE32(dictPtr);
+    cctx->dictID = cctx->appliedParams.fParams.noDictIDFlag ? 0 :  MEM_readLE32(dictPtr);
     dictPtr += 4;
 
     {   size_t const hufHeaderSize = HUF_readCTable(cctx->hufCTable, 255, dictPtr, dictEnd-dictPtr);
@@ -3053,7 +3057,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
 
     /* special case : empty frame */
     if (cctx->stage == ZSTDcs_init) {
-        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
+        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0);
         if (ZSTD_isError(fhSize)) return fhSize;
         dstCapacity -= fhSize;
         op += fhSize;
@@ -3069,7 +3073,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
         dstCapacity -= ZSTD_blockHeaderSize;
     }
 
-    if (cctx->params.fParams.checksumFlag) {
+    if (cctx->appliedParams.fParams.checksumFlag) {
         U32 const checksum = (U32) XXH64_digest(&cctx->xxhState);
         if (dstCapacity<4) return ERROR(dstSize_tooSmall);
         MEM_writeLE32(op, checksum);
@@ -3092,7 +3096,7 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
     if (ZSTD_isError(cSize)) return cSize;
     endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
     if (ZSTD_isError(endResult)) return endResult;
-    if (cctx->params.fParams.contentSizeFlag) {  /* control src size */
+    if (cctx->appliedParams.fParams.contentSizeFlag) {  /* control src size */
         if (cctx->frameContentSize != cctx->consumedSrcSize)
             return ERROR(srcSize_wrong);
     }
@@ -3269,7 +3273,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     if (cdict->dictContentSize)
         CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) )
     else {
-        ZSTD_parameters params = cdict->refContext->params;
+        ZSTD_parameters params = cdict->refContext->appliedParams;
         params.fParams = fParams;
         CHECK_F(ZSTD_compressBegin_internal(cctx, NULL, 0, params, pledgedSrcSize));
     }
@@ -3352,12 +3356,16 @@ size_t ZSTD_CStreamOutSize(void)
     return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
 }
 
-static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize)
+static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
+                                        ZSTD_parameters params,
+                                        unsigned long long pledgedSrcSize)
 {
-    DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u", !zcs->params.fParams.noDictIDFlag);
+    DEBUGLOG(5, "ZSTD_resetCStream_internal");
 
-    if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs, zcs->cdict, params.fParams, pledgedSrcSize))
-    else CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, params, pledgedSrcSize));
+    if (zcs->cdict)
+        CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs, zcs->cdict, params.fParams, pledgedSrcSize))
+    else
+        CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, params, pledgedSrcSize));
 
     zcs->inToCompress = 0;
     zcs->inBuffPos = 0;
@@ -3375,6 +3383,7 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
                                 const ZSTD_parameters params,
                                 unsigned long long pledgedSrcSize)
 {
+    DEBUGLOG(5, "ZSTD_initCStream_stage2");
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     zcs->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
 
@@ -3402,8 +3411,8 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
 
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
-
-    ZSTD_parameters params = zcs->params;
+    DEBUGLOG(5, "ZSTD_resetCStream");
+    ZSTD_parameters params = zcs->requestedParams;
     params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
     if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
         params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
@@ -3421,6 +3430,8 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
     if (!cdict) return ERROR(dictionary_wrong);
     {   ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
         params.fParams = fParams;
+        zcs->requestedParams = params;
+        zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
         zcs->cdict = cdict;
         return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
     }
@@ -3430,6 +3441,8 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 {
     ZSTD_frameParameters const fParams = { 0 /* contentSize */, 0 /* checksum */, 0 /* hideDictID */ };
+    /* cannot handle NULL cdict (does not know what to do) */
+    if (!cdict) return ERROR(dictionary_wrong);
     return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, 0, fParams);
 }
 
@@ -3455,12 +3468,15 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     CHECK_F( ZSTD_checkCParams(params.cParams) );
+    zcs->requestedParams = params;
+    zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
     return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize);
 }
 
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
+    zcs->compressionLevel = compressionLevel;
     return ZSTD_initCStream_internal(zcs, dict, dictSize, params, 0);
 }
 
@@ -3505,7 +3521,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
     while (someMoreWork) {
         switch(zcs->streamStage)
         {
-        case zcss_init: return ERROR(init_missing);   /* call ZSTD_initCStream() first ! */
+        case zcss_init:
+            /* call ZSTD_initCStream() first ! */
+            return ERROR(init_missing);
 
         case zcss_load:
             /* complete inBuffer */
@@ -3628,7 +3646,7 @@ size_t ZSTD_compress_generic_integral (
     assert(cctx!=NULL);
     if (cctx->streamStage == zcss_init) {
         /* transparent reset */
-        ZSTD_parameters params = cctx->params;
+        ZSTD_parameters params = cctx->requestedParams;
         DEBUGLOG(5, "ZSTD_compress_generic_integral : transparent reset");
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index 543761191..38c1ca620 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -43,6 +43,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t src
     if (ssPtr->litLengthSum == 0) {
         if (srcSize <= 1024) ssPtr->staticPrices = 1;
 
+        assert(ssPtr->litFreq!=NULL);
         for (u=0; u<=MaxLit; u++)
             ssPtr->litFreq[u] = 0;
         for (u=0; ubase;
     const U32 current = (U32)(ip-base);
-    const U32 hashLog = zc->params.cParams.hashLog;
+    const U32 hashLog = zc->appliedParams.cParams.hashLog;
     const size_t h  = ZSTD_hashPtr(ip, hashLog, mls);
     U32* const hashTable = zc->hashTable;
     U32 matchIndex  = hashTable[h];
     U32* const bt   = zc->chainTable;
-    const U32 btLog = zc->params.cParams.chainLog - 1;
+    const U32 btLog = zc->appliedParams.cParams.chainLog - 1;
     const U32 btMask= (1U << btLog) - 1;
     size_t commonLengthSmaller=0, commonLengthLarger=0;
     const BYTE* const dictBase = zc->dictBase;
@@ -410,10 +411,10 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
     const BYTE* const base = ctx->base;
     const BYTE* const prefixStart = base + ctx->dictLimit;
 
-    const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
-    const U32 sufficient_len = ctx->params.cParams.targetLength;
-    const U32 mls = ctx->params.cParams.searchLength;
-    const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
+    const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog;
+    const U32 sufficient_len = ctx->appliedParams.cParams.targetLength;
+    const U32 mls = ctx->appliedParams.cParams.searchLength;
+    const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;
 
     ZSTD_optimal_t* opt = seqStorePtr->priceTable;
     ZSTD_match_t* matches = seqStorePtr->matchTable;
@@ -663,10 +664,10 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
     const BYTE* const dictBase = ctx->dictBase;
     const BYTE* const dictEnd  = dictBase + dictLimit;
 
-    const U32 maxSearches = 1U << ctx->params.cParams.searchLog;
-    const U32 sufficient_len = ctx->params.cParams.targetLength;
-    const U32 mls = ctx->params.cParams.searchLength;
-    const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4;
+    const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog;
+    const U32 sufficient_len = ctx->appliedParams.cParams.targetLength;
+    const U32 mls = ctx->appliedParams.cParams.searchLength;
+    const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4;
 
     ZSTD_optimal_t* opt = seqStorePtr->priceTable;
     ZSTD_match_t* matches = seqStorePtr->matchTable;

From b0739bcf8f90ce2fbb8944293997f338d5f1db89 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 22 May 2017 17:45:15 -0700
Subject: [PATCH 018/109] simplified reset by removing full-reset policy

this was meant to be applied prior to dictionary loading.
But effectively, it seems redundant with later loading stage,
so it can be skipped safely.
---
 doc/zstd_manual.html         |  2 +-
 lib/compress/zstd_compress.c | 12 +++++-------
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index de0ee4339..c64c4290a 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -616,7 +616,7 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
  
 


 
-
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);  /* Not ready yet ! */
+
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
 
  Add a prepared dictionary to cctx, to be used for next compression jobs.
   Note that compression parameters will be enforced from within CDict.
   Currently, they supercede any compression parameter previously set within CCtx.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 0e694d2ef..be85bd49f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -478,7 +478,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
     return 0;
 }
 
-typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
+typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e;
 
 /*! ZSTD_resetCCtx_internal() :
     note : `params` must be validated */
@@ -489,13 +489,13 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
     DEBUGLOG(5, "ZSTD_resetCCtx_internal : wlog=%u / old=%u",
                 params.cParams.windowLog, zc->appliedParams.cParams.windowLog);
 
-    if (crp == ZSTDcrp_continue)
+    if (crp == ZSTDcrp_continue) {
         if (ZSTD_equivalentParams(params.cParams, zc->appliedParams.cParams)) {
             DEBUGLOG(5, "ZSTD_equivalentParams()==1");
             zc->fseCTables_ready = 0;
             zc->hufCTable_repeatMode = HUF_repeat_none;
             return ZSTD_continueCCtx(zc, params, frameContentSize);
-        }
+    }   }
 
     {   size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
         U32    const divider = (params.cParams.searchLength==3) ? 3 : 4;
@@ -631,7 +631,6 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
     {   ZSTD_parameters params = srcCCtx->appliedParams;
         params.fParams = fParams;
-        DEBUGLOG(5, "ZSTD_resetCCtx_internal : dictIDFlag : %u", !fParams.noDictIDFlag);
         ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
     }
 
@@ -3011,9 +3010,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
                                    ZSTD_parameters params, U64 pledgedSrcSize)
 {
-    ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
-    CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, crp));
+    CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue));
     return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
 }
 
@@ -3411,9 +3409,9 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
 
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
-    DEBUGLOG(5, "ZSTD_resetCStream");
     ZSTD_parameters params = zcs->requestedParams;
     params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
+    DEBUGLOG(5, "ZSTD_resetCStream");
     if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
         params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
     }

From 1880337c301de6afb60cb0401016da714f6e49c5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 22 May 2017 18:21:51 -0700
Subject: [PATCH 019/109] Simplifier compression call graph

Everything converge towards ZSTD_compressBegin_internal
which delegated to ZSTD_copyCCtx_internal if cdict!=NULL.

This simplifies routing which was previously depending on cdict.
---
 lib/compress/zstd_compress.c | 41 ++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index be85bd49f..b8648eb43 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -93,6 +93,13 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
 ***************************************/
 typedef enum { zcss_init=0, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
 
+struct ZSTD_CDict_s {
+    void* dictBuffer;
+    const void* dictContent;
+    size_t dictContentSize;
+    ZSTD_CCtx* refContext;
+};  /* typedef'd tp ZSTD_CDict within "zstd.h" */
+
 struct ZSTD_CCtx_s {
     const BYTE* nextSrc;    /* next block here to continue on current prefix */
     const BYTE* base;       /* All regular indexes relative to this position */
@@ -3008,9 +3015,15 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict,
 *   @return : 0, or an error code */
 static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
+                             const ZSTD_CDict* cdict,
                                    ZSTD_parameters params, U64 pledgedSrcSize)
 {
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
+
+    if (cdict && cdict->dictContentSize>0)
+        return ZSTD_copyCCtx_internal(cctx, cdict->refContext, params.fParams, pledgedSrcSize);
+
     CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue));
     return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
 }
@@ -3024,14 +3037,14 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
 {
     /* compression parameters verification and optimization */
     CHECK_F(ZSTD_checkCParams(params.cParams));
-    return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
+    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, params, pledgedSrcSize);
 }
 
 
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
-    return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
+    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, params, 0);
 }
 
 
@@ -3108,7 +3121,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
                          const void* dict,size_t dictSize,
                                ZSTD_parameters params)
 {
-    CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize));
+    CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, params, srcSize));
     return ZSTD_compressEnd(cctx, dst,  dstCapacity, src, srcSize);
 }
 
@@ -3149,13 +3162,6 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
 
 /* =====  Dictionary API  ===== */
 
-struct ZSTD_CDict_s {
-    void* dictBuffer;
-    const void* dictContent;
-    size_t dictContentSize;
-    ZSTD_CCtx* refContext;
-};  /* typedef'd tp ZSTD_CDict within "zstd.h" */
-
 /*! ZSTD_estimateCDictSize() :
  *  Estimate amount of memory that will be needed to create a dictionary with following arguments */
 size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
@@ -3267,15 +3273,11 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
 {
     if (cdict==NULL) return ERROR(dictionary_wrong);  /* does not support NULL cdict */
-    DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced : dictIDFlag == %u", !fParams.noDictIDFlag);
-    if (cdict->dictContentSize)
-        CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) )
-    else {
-        ZSTD_parameters params = cdict->refContext->appliedParams;
+    {   ZSTD_parameters params = cdict->refContext->appliedParams;
         params.fParams = fParams;
-        CHECK_F(ZSTD_compressBegin_internal(cctx, NULL, 0, params, pledgedSrcSize));
+        DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
+        return ZSTD_compressBegin_internal(cctx, NULL, 0, cdict, params, pledgedSrcSize);
     }
-    return 0;
 }
 
 /* ZSTD_compressBegin_usingCDict() :
@@ -3360,10 +3362,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
 {
     DEBUGLOG(5, "ZSTD_resetCStream_internal");
 
-    if (zcs->cdict)
-        CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs, zcs->cdict, params.fParams, pledgedSrcSize))
-    else
-        CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, params, pledgedSrcSize));
+    CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize));
 
     zcs->inToCompress = 0;
     zcs->inBuffPos = 0;

From 5ac72b417c438cdf36fffabb60ace0884ead500f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 23 May 2017 11:18:24 -0700
Subject: [PATCH 020/109] Buffered are now allocated inside workSpace

---
 lib/compress/zstd_compress.c | 115 +++++++++++++++++------------------
 1 file changed, 55 insertions(+), 60 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b8648eb43..0bed04fb4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -182,10 +182,6 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
     cctx->workSpace = NULL;
     ZSTD_freeCDict(cctx->cdictLocal);
     cctx->cdictLocal = NULL;
-    ZSTD_free(cctx->inBuff, cctx->customMem);
-    cctx->inBuff = NULL;
-    ZSTD_free(cctx->outBuff, cctx->customMem);
-    cctx->outBuff = NULL;
     ZSTD_free(cctx, cctx->customMem);
     return 0;   /* reserved as a potential error code in the future */
 }
@@ -487,14 +483,18 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
 
 typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e;
 
+typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
+
 /*! ZSTD_resetCCtx_internal() :
-    note : `params` must be validated */
-static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
-                                       ZSTD_parameters params, U64 frameContentSize,
-                                       ZSTD_compResetPolicy_e const crp)
+    note : `params` are assumed fully validated at this stage */
+static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
+                                      ZSTD_parameters params, U64 frameContentSize,
+                                      ZSTD_compResetPolicy_e const crp,
+                                      ZSTD_buffered_policy_e const zbuff)
 {
     DEBUGLOG(5, "ZSTD_resetCCtx_internal : wlog=%u / old=%u",
                 params.cParams.windowLog, zc->appliedParams.cParams.windowLog);
+    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
 
     if (crp == ZSTDcrp_continue) {
         if (ZSTD_equivalentParams(params.cParams, zc->appliedParams.cParams)) {
@@ -508,11 +508,15 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
         U32    const divider = (params.cParams.searchLength==3) ? 3 : 4;
         size_t const maxNbSeq = blockSize / divider;
         size_t const tokenSpace = blockSize + 11*maxNbSeq;
-        size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
+        size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ?
+                                0 : (1 << params.cParams.chainLog);
         size_t const hSize = ((size_t)1) << params.cParams.hashLog;
-        U32    const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
+        U32    const hashLog3 = (params.cParams.searchLength>3) ?
+                                0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
         size_t const h3Size = ((size_t)1) << hashLog3;
         size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
+        size_t const buffOutSize = ZSTD_compressBound(blockSize)+1;
+        size_t const buffInSize = ((size_t)1 << params.cParams.windowLog) + blockSize;
         void* ptr;
 
         /* Check if workSpace is large enough, alloc a new one if needed */
@@ -521,8 +525,13 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
                                   + entropyScratchSpace_size;
             size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<workSpaceSize < neededSpace) {
                 DEBUGLOG(5, "Need to update workSpaceSize from %uK to %uK \n",
                             (unsigned)zc->workSpaceSize>>10, (unsigned)neededSpace>>10);
@@ -607,6 +616,13 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
         zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq;
         zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq;
         zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq;
+        ptr = zc->seqStore.litStart + blockSize;
+
+        /* buffers */
+        zc->inBuffSize = buffInSize;
+        zc->inBuff = (char*)ptr;
+        zc->outBuffSize = buffOutSize;
+        zc->outBuff = zc->inBuff + buffInSize;
 
         return 0;
     }
@@ -636,9 +652,12 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
     if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
 
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
-    {   ZSTD_parameters params = srcCCtx->appliedParams;
+    {   ZSTD_buffered_policy_e const zbuff = srcCCtx->inBuffSize ?
+                        ZSTDb_buffered : ZSTDb_not_buffered;
+        ZSTD_parameters params = srcCCtx->appliedParams;
         params.fParams = fParams;
-        ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
+        ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize,
+                                ZSTDcrp_noMemset, zbuff);
     }
 
     /* copy tables */
@@ -3016,15 +3035,19 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict,
 static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
                              const ZSTD_CDict* cdict,
-                                   ZSTD_parameters params, U64 pledgedSrcSize)
+                                   ZSTD_parameters params, U64 pledgedSrcSize,
+                                   ZSTD_buffered_policy_e zbuff)
 {
+    /* params are supposed to be fully validated at this point */
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
     if (cdict && cdict->dictContentSize>0)
-        return ZSTD_copyCCtx_internal(cctx, cdict->refContext, params.fParams, pledgedSrcSize);
+        return ZSTD_copyCCtx_internal(cctx, cdict->refContext,
+                                      params.fParams, pledgedSrcSize);
 
-    CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue));
+    CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
+                                    ZSTDcrp_continue, zbuff));
     return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
 }
 
@@ -3037,14 +3060,16 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
 {
     /* compression parameters verification and optimization */
     CHECK_F(ZSTD_checkCParams(params.cParams));
-    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, params, pledgedSrcSize);
+    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
+                                    params, pledgedSrcSize, ZSTDb_not_buffered);
 }
 
 
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
-    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, params, 0);
+    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
+                                       params, 0, ZSTDb_not_buffered);
 }
 
 
@@ -3121,7 +3146,8 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
                          const void* dict,size_t dictSize,
                                ZSTD_parameters params)
 {
-    CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, params, srcSize));
+    CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
+                                        params, srcSize, ZSTDb_not_buffered));
     return ZSTD_compressEnd(cctx, dst,  dstCapacity, src, srcSize);
 }
 
@@ -3272,11 +3298,12 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
     ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
 {
-    if (cdict==NULL) return ERROR(dictionary_wrong);  /* does not support NULL cdict */
+    if (cdict==NULL) return ERROR(dictionary_wrong);
     {   ZSTD_parameters params = cdict->refContext->appliedParams;
         params.fParams = fParams;
         DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
-        return ZSTD_compressBegin_internal(cctx, NULL, 0, cdict, params, pledgedSrcSize);
+        return ZSTD_compressBegin_internal(cctx, NULL, 0, cdict,
+                                params, pledgedSrcSize, ZSTDb_not_buffered);
     }
 }
 
@@ -3362,7 +3389,8 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
 {
     DEBUGLOG(5, "ZSTD_resetCStream_internal");
 
-    CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize));
+    CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, zcs->cdict,
+                                    params, pledgedSrcSize, ZSTDb_buffered));
 
     zcs->inToCompress = 0;
     zcs->inBuffPos = 0;
@@ -3373,39 +3401,6 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
     return 0;   /* ready to go */
 }
 
-/* ZSTD_initCStream_internal() :
- * params are supposed validated at this stage
- * and zcs->cdict is supposed to be correct */
-static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
-                                const ZSTD_parameters params,
-                                unsigned long long pledgedSrcSize)
-{
-    DEBUGLOG(5, "ZSTD_initCStream_stage2");
-    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
-    zcs->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
-
-    /* allocate buffers */
-    {   size_t const neededInBuffSize = ((size_t)1 << params.cParams.windowLog) + zcs->blockSize;
-        if (zcs->inBuffSize < neededInBuffSize) {
-            zcs->inBuffSize = 0;
-            ZSTD_free(zcs->inBuff, zcs->customMem);
-            zcs->inBuff = (char*)ZSTD_malloc(neededInBuffSize, zcs->customMem);
-            if (zcs->inBuff == NULL) return ERROR(memory_allocation);
-            zcs->inBuffSize = neededInBuffSize;
-        }
-    }
-    if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) {
-        size_t const outBuffSize = ZSTD_compressBound(zcs->blockSize)+1;
-        zcs->outBuffSize = 0;
-        ZSTD_free(zcs->outBuff, zcs->customMem);
-        zcs->outBuff = (char*)ZSTD_malloc(outBuffSize, zcs->customMem);
-        if (zcs->outBuff == NULL) return ERROR(memory_allocation);
-        zcs->outBuffSize = outBuffSize;
-    }
-
-    return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
-}
-
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
 {
     ZSTD_parameters params = zcs->requestedParams;
@@ -3414,7 +3409,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
         params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
     }
-    return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
+    return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
 }
 
 /* ZSTD_initCStream_usingCDict_advanced() :
@@ -3430,7 +3425,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
         zcs->requestedParams = params;
         zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
         zcs->cdict = cdict;
-        return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
+        return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
     }
 }
 
@@ -3457,7 +3452,7 @@ static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
         zcs->cdict = zcs->cdictLocal;
     }
 
-    return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
+    return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
 }
 
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
@@ -3648,7 +3643,7 @@ size_t ZSTD_compress_generic_integral (
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
                                     cctx->frameContentSize, 0 /* dictSize */);
-        CHECK_F( ZSTD_initCStream_stage2(cctx, params, cctx->frameContentSize) );
+        CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->frameContentSize) );
     }
 
     {   size_t sizeRead = srcSize - *srcPos;

From c7fe262dc94b8253b6e0427cd89d4ccdf5d0cbaa Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 23 May 2017 13:16:00 -0700
Subject: [PATCH 021/109] added ZSTD_initStaticCCtx()

makes it possible to statically or externally allocate CCtx.
static CCtx will only use provided memory area,
it will never resize nor malloc.
---
 doc/zstd_manual.html         | 18 +++++++++-
 lib/compress/zstd_compress.c | 65 +++++++++++++++++++++++++++++-------
 lib/zstd.h                   | 18 +++++++++-
 tests/fuzzer.c               | 57 +++++++++++++++++++++++++++++++
 4 files changed, 144 insertions(+), 14 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index c64c4290a..d84f8396e 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -423,6 +423,22 @@ size_t ZSTD_estimateDDictSize(size_t dictSize);
 
  Create a ZSTD compression context using external alloc and free functions 
 


 
+
ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
+
  workspace: The memory area to emplace the context into.
+             Provided pointer must 8-bytes aligned.
+             It must outlive context usage.
+  workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
+                 to determine how large workspace must be to support scenario.
+ @return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
+  Note : zstd will never resize nor malloc() when using a static cctx.
+         If it needs more memory than available, it will simply error out.
+  Note 2 : there is no corresponding "free" function.
+           Since workspace was allocated externally, it must be freed externally too.
+  Limitation : currently not compatible with internal CDict creation, such as
+               ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
+ 
+


+
 
typedef enum {
     ZSTD_p_forceWindow,   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
     ZSTD_p_forceRawDict   /* Force loading dictionary in "content-only" mode (no header analysis) */
@@ -714,7 +730,7 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
 
 
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 0bed04fb4..deee3cfc7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -126,6 +126,7 @@ struct ZSTD_CCtx_s {
     U64 consumedSrcSize;
     XXH64_state_t xxhState;
     ZSTD_customMem customMem;
+    size_t staticSize;
 
     seqStore_t seqStore;    /* sequences storage ptrs */
     U32* hashTable;
@@ -175,9 +176,38 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
     return cctx;
 }
 
+ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
+{
+    ZSTD_CCtx* cctx = (ZSTD_CCtx*) workspace;
+    if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL;  /* minimum size */
+    if ((size_t)workspace & 7) return NULL;  /* must be 8-aligned */
+    memset(workspace, 0, workspaceSize);
+    cctx->staticSize = workspaceSize;
+    cctx->workSpace = (void*)(cctx+1);
+    cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx);
+
+    /* entropy space (never moves) */
+    /* note : this code should be shared with resetCCtx, instead of copied */
+    {   void* ptr = cctx->workSpace;
+        cctx->hufCTable = (HUF_CElt*)ptr;
+        ptr = (char*)cctx->hufCTable + hufCTable_size;  /* note : HUF_CElt* is incomplete type, size is estimated via macro */
+        cctx->offcodeCTable = (FSE_CTable*) ptr;
+        ptr = (char*)ptr + offcodeCTable_size;
+        cctx->matchlengthCTable = (FSE_CTable*) ptr;
+        ptr = (char*)ptr + matchlengthCTable_size;
+        cctx->litlengthCTable = (FSE_CTable*) ptr;
+        ptr = (char*)ptr + litlengthCTable_size;
+        assert(((size_t)ptr & 3) == 0);   /* ensure correct alignment */
+        cctx->entropyScratchSpace = (unsigned*) ptr;
+    }
+
+    return cctx;
+}
+
 size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
 {
     if (cctx==NULL) return 0;   /* support free on NULL */
+    assert(!cctx->staticSize);  /* not compatible with static CCtx */
     ZSTD_free(cctx->workSpace, cctx->customMem);
     cctx->workSpace = NULL;
     ZSTD_freeCDict(cctx->cdictLocal);
@@ -337,6 +367,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
 {
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+    if (cctx->staticSize) return ERROR(memory_allocation);  /* no malloc for static CCtx */
     ZSTD_freeCDict(cctx->cdictLocal);  /* in case one already exists */
     if (dict==NULL || dictSize==0) {   /* no dictionary mode */
         cctx->cdictLocal = NULL;
@@ -448,6 +479,17 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
     return sizeof(ZSTD_CCtx) + neededSpace;
 }
 
+size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
+{
+    size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
+    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
+    size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
+    size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
+    size_t const streamingSize = inBuffSize + outBuffSize;
+
+    return CCtxSize + streamingSize;
+}
+
 
 static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
                                  ZSTD_compressionParameters cParams2)
@@ -532,9 +574,14 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                                         buffInSize + buffOutSize : 0;
             size_t const neededSpace = entropySpace + optSpace + tableSpace
                                      + tokenSpace + bufferSpace;
-            if (zc->workSpaceSize < neededSpace) {
+
+            if (zc->workSpaceSize < neededSpace) {  /* too small : resize /*/
                 DEBUGLOG(5, "Need to update workSpaceSize from %uK to %uK \n",
-                            (unsigned)zc->workSpaceSize>>10, (unsigned)neededSpace>>10);
+                            (unsigned)zc->workSpaceSize>>10,
+                            (unsigned)neededSpace>>10);
+                /* static cctx : no resize, error out */
+                if (zc->staticSize) return ERROR(memory_allocation);
+
                 zc->workSpaceSize = 0;
                 ZSTD_free(zc->workSpace, zc->customMem);
                 zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
@@ -3362,16 +3409,6 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
     return ZSTD_freeCCtx(zcs);   /* same object */
 }
 
-size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
-{
-    size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
-    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
-    size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
-    size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
-    size_t const streamingSize = inBuffSize + outBuffSize;
-
-    return CCtxSize + streamingSize;
-}
 
 
 /*======   Initialization   ======*/
@@ -3446,6 +3483,10 @@ static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
     zcs->cdict = NULL;
 
     if (dict && dictSize >= 8) {
+        if (zcs->staticSize) {   /* static CCtx : never uses malloc */
+            /* incompatible with internal cdict creation */
+            return ERROR(memory_allocation);
+        }
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params.cParams, zcs->customMem);
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
diff --git a/lib/zstd.h b/lib/zstd.h
index 83fe9d30f..d25130faf 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -511,6 +511,22 @@ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
  *  Create a ZSTD compression context using external alloc and free functions */
 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 
+/*! ZSTD_initStaticCCtx() : initialize a fixed-size zstd compression context
+ *  workspace: The memory area to emplace the context into.
+ *             Provided pointer must 8-bytes aligned.
+ *             It must outlive context usage.
+ *  workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
+ *                 to determine how large workspace must be to support scenario.
+ * @return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
+ *  Note : zstd will never resize nor malloc() when using a static cctx.
+ *         If it needs more memory than available, it will simply error out.
+ *  Note 2 : there is no corresponding "free" function.
+ *           Since workspace was allocated externally, it must be freed externally too.
+ *  Limitation : currently not compatible with internal CDict creation, such as
+ *               ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
+ */
+ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
+
 
 /* !!! Soon to be deprecated !!! */
 typedef enum {
@@ -841,7 +857,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 /*=====   Advanced Streaming compression functions  =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
-ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */
+ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 583018a45..a1dc6ba77 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -190,6 +190,63 @@ static int basicUnitTests(U32 seed, double compressibility)
     DISPLAYLEVEL(4, "OK \n");
 
 
+    /* Static CCtx tests */
+#define STATIC_CCTX_LEVEL 3
+    DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
+    {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(STATIC_CCTX_LEVEL, 0, 0);
+        size_t const staticCCtxSize = ZSTD_estimateCStreamSize(cParams);
+        void* staticCCtxBuffer = malloc(staticCCtxSize);
+        if (staticCCtxBuffer==NULL) {
+            DISPLAY("Not enough memory, aborting\n");
+            testResult = 1;
+            goto _end;
+        }
+        {   ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
+            if (staticCCtx==NULL) goto _output_error;
+            DISPLAYLEVEL(4, "OK \n");
+
+            DISPLAYLEVEL(4, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
+            { size_t const r = ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL);
+              if (ZSTD_isError(r)) goto _output_error; }
+            DISPLAYLEVEL(4, "OK \n");
+
+            DISPLAYLEVEL(4, "test%3i : simple compression test with static CCtx : ", testNb++);
+            CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx,
+                            compressedBuffer, ZSTD_compressBound(CNBuffSize),
+                            CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL),
+                      cSize=r );
+            DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+            DISPLAYLEVEL(4, "test%3i : decompress verification test : ", testNb++);
+            { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+              if (r != CNBuffSize) goto _output_error; }
+            DISPLAYLEVEL(4, "OK \n");
+
+            DISPLAYLEVEL(4, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
+            { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
+              if (!ZSTD_isError(r)) goto _output_error; }
+            DISPLAYLEVEL(4, "OK \n");
+
+            DISPLAYLEVEL(4, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
+            { size_t const r = ZSTD_compressBegin(staticCCtx, 1);
+              if (ZSTD_isError(r)) goto _output_error; }
+            DISPLAYLEVEL(4, "OK \n");
+
+            DISPLAYLEVEL(4, "test%3i : init CStream for small level %u : ", testNb++, 1);
+            { size_t const r = ZSTD_initCStream(staticCCtx, 1);
+              if (ZSTD_isError(r)) goto _output_error; }
+            DISPLAYLEVEL(4, "OK \n");
+
+            DISPLAYLEVEL(4, "test%3i : init CStream with dictionary (should fail) : ", testNb++);
+            { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
+              if (!ZSTD_isError(r)) goto _output_error; }
+            DISPLAYLEVEL(4, "OK \n");
+        }
+        free(staticCCtxBuffer);
+    }
+
+
+
     /* ZSTDMT simple MT compression test */
     DISPLAYLEVEL(4, "test%3i : create ZSTDMT CCtx : ", testNb++);
     {   ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);

From b81f19ffce2ce24b6d8a13a61ec554373a426d35 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 23 May 2017 15:41:55 -0700
Subject: [PATCH 022/109] move MEM_readMINMATCH() into zstd_opt.h

which is its only user.
Use case too narrow to belong to mem.h.
renamed to ZSTD_readMINMATCH()
---
 lib/common/mem.h        | 14 --------------
 lib/compress/zstd_opt.h | 24 +++++++++++++++++++-----
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/lib/common/mem.h b/lib/common/mem.h
index 4773a8b93..b0e5bf60b 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -352,20 +352,6 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
 }
 
 
-/* function safe only for comparisons */
-MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
-{
-    switch (length)
-    {
-    default :
-    case 4 : return MEM_read32(memPtr);
-    case 3 : if (MEM_isLittleEndian())
-                return MEM_read32(memPtr)<<8;
-             else
-                return MEM_read32(memPtr)>>8;
-    }
-}
-
 #if defined (__cplusplus)
 }
 #endif
diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h
index 38c1ca620..e8e98915e 100644
--- a/lib/compress/zstd_opt.h
+++ b/lib/compress/zstd_opt.h
@@ -202,6 +202,20 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
     }
 
 
+/* function safe only for comparisons */
+MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
+{
+    switch (length)
+    {
+    default :
+    case 4 : return MEM_read32(memPtr);
+    case 3 : if (MEM_isLittleEndian())
+                return MEM_read32(memPtr)<<8;
+             else
+                return MEM_read32(memPtr)>>8;
+    }
+}
+
 
 /* Update hashTable3 up to ip (excluded)
    Assumption : always within prefix (i.e. not within extDict) */
@@ -268,7 +282,7 @@ static U32 ZSTD_insertBtAndGetAllMatches (
                 if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit);
             } else {
                 match = dictBase + matchIndex3;
-                if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH))    /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
+                if (ZSTD_readMINMATCH(match, MINMATCH) == ZSTD_readMINMATCH(ip, MINMATCH))    /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */
                     currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH;
             }
 
@@ -440,7 +454,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
             for (i=(ip == anchor); i 0) && (repCur < (S32)(ip-prefixStart))
-                    && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) {
+                    && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) {
                     mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch;
                     if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
                         best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
@@ -525,7 +539,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                 for (i=(opt[cur].mlen != 1); i 0) && (repCur < (S32)(inr-prefixStart))
-                       && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) {
+                       && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) {
                        mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch;
 
                        if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
@@ -699,7 +713,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                 const BYTE* const repMatch = repBase + repIndex;
                 if ( (repCur > 0 && repCur <= (S32)current)
                    && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex))  /* intentional overflow */
-                   && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
+                   && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
                     /* repcode detected we should take it */
                     const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
                     mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
@@ -795,7 +809,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                     const BYTE* const repMatch = repBase + repIndex;
                     if ( (repCur > 0 && repCur <= (S32)(current+cur))
                       && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex))  /* intentional overflow */
-                      && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
+                      && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
                         /* repcode detected */
                         const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
                         mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;

From 11ea2f7fda3a9508438066dab6c333210c328ea2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 23 May 2017 16:19:43 -0700
Subject: [PATCH 023/109] Merged ZSTD_DCtx and ZSTD_DStream objects

They are now the same object.
It's recommended to keep both types in source code
as previous versions of library (   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
 #  define ZSTD_PREFETCH(ptr)   _mm_prefetch((const char*)ptr, _MM_HINT_T0)
@@ -63,13 +62,27 @@
 #  define ZSTD_PREFETCH(ptr)   /* disabled */
 #endif
 
+
 /*-*************************************
-*  Macros
+*  Errors
 ***************************************/
 #define ZSTD_isError ERR_isError   /* for inlining */
 #define FSE_isError  ERR_isError
 #define HUF_isError  ERR_isError
 
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
+#  include 
+   static unsigned g_debugLevel = ZSTD_DEBUG;
+#  define DEBUGLOG(l, ...) {                          \
+                if (l<=g_debugLevel) {                \
+                    fprintf(stderr, __FILE__ ": ");   \
+                    fprintf(stderr, __VA_ARGS__);     \
+                    fprintf(stderr, " \n");           \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
+#endif
+
 
 /*_*******************************************************
 *  Memory operations
@@ -85,6 +98,9 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
                ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
                ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
 
+typedef enum { zdss_init=0, zdss_loadHeader,
+               zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
+
 typedef struct {
     FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
     FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
@@ -119,9 +135,34 @@ struct ZSTD_DCtx_s
     size_t rleSize;
     BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+
+    /* streaming */
+    ZSTD_DDict* ddictLocal;
+    const ZSTD_DDict* ddict;
+    ZSTD_dStreamStage streamStage;
+    char*  inBuff;
+    size_t inBuffSize;
+    size_t inPos;
+    size_t maxWindowSize;
+    char*  outBuff;
+    size_t outBuffSize;
+    size_t outStart;
+    size_t outEnd;
+    size_t blockSize;
+    size_t lhSize;
+    void* legacyContext;
+    U32 previousLegacyVersion;
+    U32 legacyVersion;
+    U32 hostageByte;
 };  /* typedef'd to ZSTD_DCtx within "zstd.h" */
 
-size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return (dctx==NULL) ? 0 : sizeof(ZSTD_DCtx); }
+size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
+{
+    if (dctx==NULL) return 0;   /* support sizeof NULL */
+    return sizeof(*dctx)
+           + ZSTD_sizeof_DDict(dctx->ddictLocal)
+           + dctx->inBuffSize + dctx->outBuffSize;
+}
 
 size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
 
@@ -154,8 +195,11 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
 
     dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
     if (!dctx) return NULL;
+    memset(dctx, 0, sizeof(*dctx));
     memcpy(&dctx->customMem, &customMem, sizeof(customMem));
-    ZSTD_decompressBegin(dctx);
+    ZSTD_decompressBegin(dctx);   /* cannot fail */
+    dctx->streamStage = zdss_init;
+    dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
     return dctx;
 }
 
@@ -167,8 +211,20 @@ ZSTD_DCtx* ZSTD_createDCtx(void)
 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
 {
     if (dctx==NULL) return 0;   /* support free on NULL */
-    ZSTD_free(dctx, dctx->customMem);
-    return 0;   /* reserved as a potential error code in the future */
+    {   ZSTD_customMem const cMem = dctx->customMem;
+        ZSTD_freeDDict(dctx->ddictLocal);
+        dctx->ddictLocal = NULL;
+        ZSTD_free(dctx->inBuff, cMem);
+        dctx->inBuff = NULL;
+        ZSTD_free(dctx->outBuff, cMem);
+        dctx->outBuff = NULL;
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+        if (dctx->legacyContext)
+            ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
+#endif
+        ZSTD_free(dctx, cMem);
+        return 0;
+    }
 }
 
 void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
@@ -230,7 +286,8 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
     if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
         if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
             /* skippable frame */
-            if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
+            if (srcSize < ZSTD_skippableHeaderSize)
+                return ZSTD_skippableHeaderSize; /* magic number + frame length */
             memset(zfhPtr, 0, sizeof(*zfhPtr));
             zfhPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
             zfhPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
@@ -253,11 +310,13 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
         U32 windowSize = 0;
         U32 dictID = 0;
         U64 frameContentSize = 0;
-        if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported);   /* reserved bits, which must be zero */
+        if ((fhdByte & 0x08) != 0)
+            return ERROR(frameParameter_unsupported);   /* reserved bits, must be zero */
         if (!singleSegment) {
             BYTE const wlByte = ip[pos++];
             U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
-            if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge);  /* avoids issue with 1 << windowLog */
+            if (windowLog > ZSTD_WINDOWLOG_MAX)
+                return ERROR(frameParameter_windowTooLarge);
             windowSize = (1U << windowLog);
             windowSize += (windowSize >> 3) * (wlByte&7);
         }
@@ -321,51 +380,48 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
  *  @return : decompressed size of the frames contained */
 unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
 {
-    {
-        unsigned long long totalDstSize = 0;
-        while (srcSize >= ZSTD_frameHeaderSize_prefix) {
-            const U32 magicNumber = MEM_readLE32(src);
+    unsigned long long totalDstSize = 0;
 
-            if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
-                size_t skippableSize;
-                if (srcSize < ZSTD_skippableHeaderSize)
-                    return ERROR(srcSize_wrong);
-                skippableSize = MEM_readLE32((const BYTE *)src + 4) +
-                                ZSTD_skippableHeaderSize;
-                if (srcSize < skippableSize) {
-                    return ZSTD_CONTENTSIZE_ERROR;
-                }
+    while (srcSize >= ZSTD_frameHeaderSize_prefix) {
+        const U32 magicNumber = MEM_readLE32(src);
 
-                src = (const BYTE *)src + skippableSize;
-                srcSize -= skippableSize;
-                continue;
+        if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+            size_t skippableSize;
+            if (srcSize < ZSTD_skippableHeaderSize)
+                return ERROR(srcSize_wrong);
+            skippableSize = MEM_readLE32((const BYTE *)src + 4) +
+                            ZSTD_skippableHeaderSize;
+            if (srcSize < skippableSize) {
+                return ZSTD_CONTENTSIZE_ERROR;
             }
 
-            {
-                unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
-                if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
-
-                /* check for overflow */
-                if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
-                totalDstSize += ret;
-            }
-            {
-                size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
-                if (ZSTD_isError(frameSrcSize)) {
-                    return ZSTD_CONTENTSIZE_ERROR;
-                }
-
-                src = (const BYTE *)src + frameSrcSize;
-                srcSize -= frameSrcSize;
-            }
+            src = (const BYTE *)src + skippableSize;
+            srcSize -= skippableSize;
+            continue;
         }
 
-        if (srcSize) {
-            return ZSTD_CONTENTSIZE_ERROR;
-        }
+        {   unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
+            if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
 
-        return totalDstSize;
+            /* check for overflow */
+            if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
+            totalDstSize += ret;
+        }
+        {   size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
+            if (ZSTD_isError(frameSrcSize)) {
+                return ZSTD_CONTENTSIZE_ERROR;
+            }
+
+            src = (const BYTE *)src + frameSrcSize;
+            srcSize -= frameSrcSize;
+        }
     }
+
+    if (srcSize) {
+        return ZSTD_CONTENTSIZE_ERROR;
+    }
+
+    return totalDstSize;
 }
 
 /** ZSTD_getDecompressedSize() :
@@ -2044,35 +2100,6 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
 *   Streaming decompression
 *====================================*/
 
-typedef enum { zdss_init, zdss_loadHeader,
-               zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
-
-/* *** Resource management *** */
-struct ZSTD_DStream_s {
-    ZSTD_DCtx* dctx;
-    ZSTD_DDict* ddictLocal;
-    const ZSTD_DDict* ddict;
-    ZSTD_frameHeader fParams;
-    ZSTD_dStreamStage stage;
-    char*  inBuff;
-    size_t inBuffSize;
-    size_t inPos;
-    size_t maxWindowSize;
-    char*  outBuff;
-    size_t outBuffSize;
-    size_t outStart;
-    size_t outEnd;
-    size_t blockSize;
-    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];   /* tmp buffer to store frame header */
-    size_t lhSize;
-    ZSTD_customMem customMem;
-    void* legacyContext;
-    U32 previousLegacyVersion;
-    U32 legacyVersion;
-    U32 hostageByte;
-};   /* typedef'd to ZSTD_DStream within "zstd.h" */
-
-
 ZSTD_DStream* ZSTD_createDStream(void)
 {
     return ZSTD_createDStream_advanced(defaultCustomMem);
@@ -2080,41 +2107,12 @@ ZSTD_DStream* ZSTD_createDStream(void)
 
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
 {
-    ZSTD_DStream* zds;
-
-    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-    if (!customMem.customAlloc || !customMem.customFree) return NULL;
-
-    zds = (ZSTD_DStream*) ZSTD_malloc(sizeof(ZSTD_DStream), customMem);
-    if (zds==NULL) return NULL;
-    memset(zds, 0, sizeof(ZSTD_DStream));
-    memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
-    zds->dctx = ZSTD_createDCtx_advanced(customMem);
-    if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; }
-    zds->stage = zdss_init;
-    zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
-    return zds;
+    return ZSTD_createDCtx_advanced(customMem);
 }
 
 size_t ZSTD_freeDStream(ZSTD_DStream* zds)
 {
-    if (zds==NULL) return 0;   /* support free on null */
-    {   ZSTD_customMem const cMem = zds->customMem;
-        ZSTD_freeDCtx(zds->dctx);
-        zds->dctx = NULL;
-        ZSTD_freeDDict(zds->ddictLocal);
-        zds->ddictLocal = NULL;
-        ZSTD_free(zds->inBuff, cMem);
-        zds->inBuff = NULL;
-        ZSTD_free(zds->outBuff, cMem);
-        zds->outBuff = NULL;
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
-        if (zds->legacyContext)
-            ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion);
-#endif
-        ZSTD_free(zds, cMem);
-        return 0;
-    }
+    return ZSTD_freeDCtx(zds);
 }
 
 
@@ -2125,7 +2123,7 @@ size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
 
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
 {
-    zds->stage = zdss_loadHeader;
+    zds->streamStage = zdss_loadHeader;
     zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
     ZSTD_freeDDict(zds->ddictLocal);
     if (dict && dictSize >= 8) {
@@ -2154,7 +2152,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict)
 
 size_t ZSTD_resetDStream(ZSTD_DStream* zds)
 {
-    zds->stage = zdss_loadHeader;
+    zds->streamStage = zdss_loadHeader;
     zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
     zds->legacyVersion = 0;
     zds->hostageByte = 0;
@@ -2175,11 +2173,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
 
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
 {
-    if (zds==NULL) return 0;   /* support sizeof NULL */
-    return sizeof(*zds)
-           + ZSTD_sizeof_DCtx(zds->dctx)
-           + ZSTD_sizeof_DDict(zds->ddictLocal)
-           + zds->inBuffSize + zds->outBuffSize;
+    return ZSTD_sizeof_DCtx(zds);
 }
 
 size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader)
@@ -2218,7 +2212,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 #endif
 
     while (someMoreWork) {
-        switch(zds->stage)
+        switch(zds->streamStage)
         {
         case zdss_init :
             ZSTD_resetDStream(zds);   /* transparent reset on starting decoding a new frame */
@@ -2259,22 +2253,22 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
                 size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
                 if (cSize <= (size_t)(iend-istart)) {
-                    size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict);
+                    size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict);
                     if (ZSTD_isError(decompressedSize)) return decompressedSize;
                     ip = istart + cSize;
                     op += decompressedSize;
-                    zds->dctx->expected = 0;
-                    zds->stage = zdss_init;
+                    zds->expected = 0;
+                    zds->streamStage = zdss_init;
                     someMoreWork = 0;
                     break;
             }   }
 
             /* Consume header */
-            CHECK_F(ZSTD_decompressBegin_usingDDict(zds->dctx, zds->ddict));
-            {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);  /* == ZSTD_frameHeaderSize_prefix */
-                CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
-                {   size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
-                    CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size));
+            CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
+            {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds);  /* == ZSTD_frameHeaderSize_prefix */
+                CHECK_F(ZSTD_decompressContinue(zds, NULL, 0, zds->headerBuffer, h1Size));
+                {   size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds);
+                    CHECK_F(ZSTD_decompressContinue(zds, NULL, 0, zds->headerBuffer+h1Size, h2Size));
             }   }
 
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
@@ -2285,6 +2279,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
                 zds->blockSize = blockSize;
                 if (zds->inBuffSize < blockSize) {
+                    DEBUGLOG(4, "inBuff  : from %u to %u",
+                                (U32)zds->inBuffSize, (U32)blockSize);
                     ZSTD_free(zds->inBuff, zds->customMem);
                     zds->inBuffSize = 0;
                     zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem);
@@ -2292,41 +2288,43 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     zds->inBuffSize = blockSize;
                 }
                 if (zds->outBuffSize < neededOutSize) {
+                    DEBUGLOG(4, "outBuff : from %u to %u",
+                                (U32)zds->outBuffSize, (U32)neededOutSize);
                     ZSTD_free(zds->outBuff, zds->customMem);
                     zds->outBuffSize = 0;
                     zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem);
                     if (zds->outBuff == NULL) return ERROR(memory_allocation);
                     zds->outBuffSize = neededOutSize;
             }   }
-            zds->stage = zdss_read;
+            zds->streamStage = zdss_read;
             /* pass-through */
 
         case zdss_read:
-            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
+            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
                 if (neededInSize==0) {  /* end of frame */
-                    zds->stage = zdss_init;
+                    zds->streamStage = zdss_init;
                     someMoreWork = 0;
                     break;
                 }
                 if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
-                    const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
-                    size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
+                    const int isSkipFrame = ZSTD_isSkipFrame(zds);
+                    size_t const decodedSize = ZSTD_decompressContinue(zds,
                         zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
                         ip, neededInSize);
                     if (ZSTD_isError(decodedSize)) return decodedSize;
                     ip += neededInSize;
                     if (!decodedSize && !isSkipFrame) break;   /* this was just a header */
                     zds->outEnd = zds->outStart + decodedSize;
-                    zds->stage = zdss_flush;
+                    zds->streamStage = zdss_flush;
                     break;
                 }
                 if (ip==iend) { someMoreWork = 0; break; }   /* no more input */
-                zds->stage = zdss_load;
+                zds->streamStage = zdss_load;
                 /* pass-through */
             }
 
         case zdss_load:
-            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
+            {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
                 size_t const toLoad = neededInSize - zds->inPos;   /* should always be <= remaining space within inBuff */
                 size_t loadedSize;
                 if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected);   /* should never happen */
@@ -2336,15 +2334,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */
 
                 /* decode loaded input */
-                {  const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
-                   size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
+                {  const int isSkipFrame = ZSTD_isSkipFrame(zds);
+                   size_t const decodedSize = ZSTD_decompressContinue(zds,
                         zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
                         zds->inBuff, neededInSize);
                     if (ZSTD_isError(decodedSize)) return decodedSize;
                     zds->inPos = 0;   /* input is consumed */
-                    if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; }   /* this was just a header */
+                    if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; }   /* this was just a header */
                     zds->outEnd = zds->outStart +  decodedSize;
-                    zds->stage = zdss_flush;
+                    zds->streamStage = zdss_flush;
                     /* pass-through */
             }   }
 
@@ -2354,7 +2352,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 op += flushedSize;
                 zds->outStart += flushedSize;
                 if (flushedSize == toFlushSize) {  /* flush completed */
-                    zds->stage = zdss_read;
+                    zds->streamStage = zdss_read;
                     if (zds->outStart + zds->blockSize > zds->outBuffSize)
                         zds->outStart = zds->outEnd = 0;
                     break;
@@ -2369,11 +2367,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     /* result */
     input->pos += (size_t)(ip-istart);
     output->pos += (size_t)(op-ostart);
-    {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
+    {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds);
         if (!nextSrcSizeHint) {   /* frame fully decoded */
             if (zds->outEnd == zds->outStart) {  /* output fully flushed */
                 if (zds->hostageByte) {
-                    if (input->pos >= input->size) { zds->stage = zdss_read; return 1; }  /* can't release hostage (not present) */
+                    if (input->pos >= input->size) {
+                        /* can't release hostage (not present) */
+                        zds->streamStage = zdss_read;
+                        return 1;
+                    }
                     input->pos++;  /* release hostage */
                 }
                 return 0;
@@ -2384,7 +2386,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }
             return 1;
         }
-        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block);   /* preload header of next block */
+        nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block);   /* preload header of next block */
         if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC);   /* should never happen */
         nextSrcSizeHint -= zds->inPos;   /* already loaded*/
         return nextSrcSizeHint;
diff --git a/lib/zstd.h b/lib/zstd.h
index d25130faf..81f9eb737 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -329,7 +329,8 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 *            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
 * *******************************************************************************/
 
-typedef struct ZSTD_DStream_s ZSTD_DStream;
+//typedef struct ZSTD_DStream_s ZSTD_DStream;
+typedef ZSTD_DCtx ZSTD_DStream;
 /*===== ZSTD_DStream management functions =====*/
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);

From 2e4db3e531e224118a316d50174300116a2f99ca Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 24 May 2017 13:15:19 -0700
Subject: [PATCH 024/109] fixed performance regression with ZSTD_decompress()
 on small files

memset() was a quick fix to initialization problems,
but initialize too much space (tables, buffers)
which show up in decompression speed of ZSTD_decompress()
since it needs to recreate DCtx at each invocation.

Fixed by only initialization relevant pointers and size fields.
---
 lib/decompress/zstd_decompress.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index aabfa6d94..8cf500445 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -195,11 +195,16 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
 
     dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
     if (!dctx) return NULL;
-    memset(dctx, 0, sizeof(*dctx));
     memcpy(&dctx->customMem, &customMem, sizeof(customMem));
     ZSTD_decompressBegin(dctx);   /* cannot fail */
     dctx->streamStage = zdss_init;
     dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
+    dctx->ddict   = NULL;
+    dctx->ddictLocal = NULL;
+    dctx->inBuff  = NULL;
+    dctx->outBuff = NULL;
+    dctx->inBuffSize = 0;
+    dctx->outBuffSize= 0;
     return dctx;
 }
 

From ba183005d340daf28a39f11e92d422e1831f94a1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 24 May 2017 15:42:24 -0700
Subject: [PATCH 025/109] merged DStream's inBuff and outBuff into a single
 buffer

Saves one malloc().
Also : makes it easier to implement static allocation
---
 lib/decompress/zstd_decompress.c | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 8cf500445..1552fec3f 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -202,7 +202,6 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
     dctx->ddict   = NULL;
     dctx->ddictLocal = NULL;
     dctx->inBuff  = NULL;
-    dctx->outBuff = NULL;
     dctx->inBuffSize = 0;
     dctx->outBuffSize= 0;
     return dctx;
@@ -221,8 +220,6 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
         dctx->ddictLocal = NULL;
         ZSTD_free(dctx->inBuff, cMem);
         dctx->inBuff = NULL;
-        ZSTD_free(dctx->outBuff, cMem);
-        dctx->outBuff = NULL;
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
         if (dctx->legacyContext)
             ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
@@ -2225,9 +2222,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
         case zdss_loadHeader :
             {   size_t const hSize = ZSTD_getFrameHeader(&zds->fParams, zds->headerBuffer, zds->lhSize);
-                if (ZSTD_isError(hSize))
+                if (ZSTD_isError(hSize)) {
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
-                {   U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
+                    U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
                     if (legacyVersion) {
                         const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
                         size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
@@ -2237,10 +2234,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                         return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
                     } else {
                         return hSize; /* error */
-                }   }
+                    }
 #else
-                return hSize;
+                    return hSize;
 #endif
+                }
                 if (hSize != 0) {   /* need more input */
                     size_t const toLoad = hSize - zds->lhSize;   /* if hSize!=0, hSize > zds->lhSize */
                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
@@ -2283,22 +2281,19 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             {   size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
                 size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2;
                 zds->blockSize = blockSize;
-                if (zds->inBuffSize < blockSize) {
+                if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) {
+                    size_t const bufferSize = blockSize + neededOutSize;
                     DEBUGLOG(4, "inBuff  : from %u to %u",
                                 (U32)zds->inBuffSize, (U32)blockSize);
-                    ZSTD_free(zds->inBuff, zds->customMem);
-                    zds->inBuffSize = 0;
-                    zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem);
-                    if (zds->inBuff == NULL) return ERROR(memory_allocation);
-                    zds->inBuffSize = blockSize;
-                }
-                if (zds->outBuffSize < neededOutSize) {
                     DEBUGLOG(4, "outBuff : from %u to %u",
                                 (U32)zds->outBuffSize, (U32)neededOutSize);
-                    ZSTD_free(zds->outBuff, zds->customMem);
+                    ZSTD_free(zds->inBuff, zds->customMem);
+                    zds->inBuffSize = 0;
                     zds->outBuffSize = 0;
-                    zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem);
-                    if (zds->outBuff == NULL) return ERROR(memory_allocation);
+                    zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
+                    if (zds->inBuff == NULL) return ERROR(memory_allocation);
+                    zds->inBuffSize = blockSize;
+                    zds->outBuff = zds->inBuff + zds->inBuffSize;
                     zds->outBuffSize = neededOutSize;
             }   }
             zds->streamStage = zdss_read;

From 0fdc71c3dc1e8cbef7416218c46d39ad0631381a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 24 May 2017 17:41:41 -0700
Subject: [PATCH 026/109] added ZSTD_initStaticDCtx()

---
 doc/zstd_manual.html             |  16 +++++
 lib/compress/zstd_compress.c     |   2 +-
 lib/decompress/zstd_decompress.c | 103 ++++++++++++++++++++-----------
 lib/zstd.h                       |  18 +++++-
 tests/fuzzer.c                   |  41 +++++++++---
 5 files changed, 135 insertions(+), 45 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index d84f8396e..72da13650 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -691,6 +691,22 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
 
  Create a ZSTD decompression context using external alloc and free functions 
 


 
+
ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
+
  workspace: The memory area to emplace the context into.
+             Provided pointer must 8-bytes aligned.
+             It must outlive context usage.
+  workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
+                 to determine how large workspace must be to support scenario.
+ @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
+  Note : zstd will never resize nor malloc() when using a static dctx.
+         If it needs more memory than available, it will simply error out.
+  Note 2 : there is no corresponding "free" function.
+           Since workspace was allocated externally, it must be freed externally.
+  Limitation : currently not compatible with internal DDict creation,
+               such as ZSTD_initDStream_usingDict().
+ 
+


+
 
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
 
  Create a digested dictionary, ready to start decompression operation without startup delay.
   Dictionary content is simply referenced, and therefore stays in dictBuffer.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index deee3cfc7..58ed98e9b 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -190,7 +190,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
     /* note : this code should be shared with resetCCtx, instead of copied */
     {   void* ptr = cctx->workSpace;
         cctx->hufCTable = (HUF_CElt*)ptr;
-        ptr = (char*)cctx->hufCTable + hufCTable_size;  /* note : HUF_CElt* is incomplete type, size is estimated via macro */
+        ptr = (char*)cctx->hufCTable + hufCTable_size;
         cctx->offcodeCTable = (FSE_CTable*) ptr;
         ptr = (char*)ptr + offcodeCTable_size;
         cctx->matchlengthCTable = (FSE_CTable*) ptr;
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 1552fec3f..2db493e51 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -133,8 +133,7 @@ struct ZSTD_DCtx_s
     ZSTD_customMem customMem;
     size_t litSize;
     size_t rleSize;
-    BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
-    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
+    size_t staticSize;
 
     /* streaming */
     ZSTD_DDict* ddictLocal;
@@ -154,6 +153,10 @@ struct ZSTD_DCtx_s
     U32 previousLegacyVersion;
     U32 legacyVersion;
     U32 hostageByte;
+
+    /* workspace */
+    BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
+    BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
 };  /* typedef'd to ZSTD_DCtx within "zstd.h" */
 
 size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
@@ -186,24 +189,45 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
     return 0;
 }
 
-ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
+static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
 {
-    ZSTD_DCtx* dctx;
-
-    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-    if (!customMem.customAlloc || !customMem.customFree) return NULL;
-
-    dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
-    if (!dctx) return NULL;
-    memcpy(&dctx->customMem, &customMem, sizeof(customMem));
     ZSTD_decompressBegin(dctx);   /* cannot fail */
-    dctx->streamStage = zdss_init;
+    dctx->staticSize = 0;
     dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
     dctx->ddict   = NULL;
     dctx->ddictLocal = NULL;
     dctx->inBuff  = NULL;
     dctx->inBuffSize = 0;
     dctx->outBuffSize= 0;
+    dctx->streamStage = zdss_init;
+}
+
+ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
+{
+    ZSTD_DCtx* dctx;
+
+    if (!customMem.customAlloc && !customMem.customFree)
+        customMem = defaultCustomMem;
+    if (!customMem.customAlloc || !customMem.customFree)
+        return NULL;
+
+    dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
+    if (!dctx) return NULL;
+    memcpy(&dctx->customMem, &customMem, sizeof(customMem));
+    ZSTD_initDCtx_internal(dctx);
+    return dctx;
+}
+
+ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
+{
+    ZSTD_DCtx* dctx = (ZSTD_DCtx*) workspace;
+
+    if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
+    if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL;  /* minimum size */
+
+    ZSTD_initDCtx_internal(dctx);
+    dctx->staticSize = workspaceSize;
+    dctx->inBuff = (char*)(dctx+1);
     return dctx;
 }
 
@@ -229,10 +253,11 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
     }
 }
 
+/* no longer appropriate */
 void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 {
-    size_t const workSpaceSize = (ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
-    memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize);  /* no need to copy workspace */
+    size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
+    memcpy(dstDCtx, srcDCtx, toCopy);  /* no need to copy workspace */
 }
 
 
@@ -2283,15 +2308,22 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 zds->blockSize = blockSize;
                 if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) {
                     size_t const bufferSize = blockSize + neededOutSize;
-                    DEBUGLOG(4, "inBuff  : from %u to %u",
+                    DEBUGLOG(5, "inBuff  : from %u to %u",
                                 (U32)zds->inBuffSize, (U32)blockSize);
-                    DEBUGLOG(4, "outBuff : from %u to %u",
+                    DEBUGLOG(5, "outBuff : from %u to %u",
                                 (U32)zds->outBuffSize, (U32)neededOutSize);
-                    ZSTD_free(zds->inBuff, zds->customMem);
-                    zds->inBuffSize = 0;
-                    zds->outBuffSize = 0;
-                    zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
-                    if (zds->inBuff == NULL) return ERROR(memory_allocation);
+                    if (zds->staticSize) {  /* static DCtx */
+                        DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
+                        assert(zds->staticSize >= sizeof(ZSTD_DCtx));  /* already checked at init */
+                        if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx))
+                            return ERROR(memory_allocation);
+                    } else {
+                        ZSTD_free(zds->inBuff, zds->customMem);
+                        zds->inBuffSize = 0;
+                        zds->outBuffSize = 0;
+                        zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
+                        if (zds->inBuff == NULL) return ERROR(memory_allocation);
+                    }
                     zds->inBuffSize = blockSize;
                     zds->outBuff = zds->inBuff + zds->inBuffSize;
                     zds->outBuffSize = neededOutSize;
@@ -2317,11 +2349,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     zds->outEnd = zds->outStart + decodedSize;
                     zds->streamStage = zdss_flush;
                     break;
-                }
-                if (ip==iend) { someMoreWork = 0; break; }   /* no more input */
-                zds->streamStage = zdss_load;
-                /* pass-through */
-            }
+            }   }
+            if (ip==iend) { someMoreWork = 0; break; }   /* no more input */
+            zds->streamStage = zdss_load;
+            /* pass-through */
 
         case zdss_load:
             {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
@@ -2342,9 +2373,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     zds->inPos = 0;   /* input is consumed */
                     if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; }   /* this was just a header */
                     zds->outEnd = zds->outStart +  decodedSize;
-                    zds->streamStage = zdss_flush;
-                    /* pass-through */
             }   }
+            zds->streamStage = zdss_flush;
+            /* pass-through */
 
         case zdss_flush:
             {   size_t const toFlushSize = zds->outEnd - zds->outStart;
@@ -2356,11 +2387,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     if (zds->outStart + zds->blockSize > zds->outBuffSize)
                         zds->outStart = zds->outEnd = 0;
                     break;
-                }
-                /* cannot complete flush */
-                someMoreWork = 0;
-                break;
-            }
+            }   }
+            /* cannot complete flush */
+            someMoreWork = 0;
+            break;
+
         default: return ERROR(GENERIC);   /* impossible */
     }   }
 
@@ -2377,15 +2408,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                         return 1;
                     }
                     input->pos++;  /* release hostage */
-                }
+                }   /* zds->hostageByte */
                 return 0;
-            }
+            }  /* zds->outEnd == zds->outStart */
             if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
                 input->pos--;   /* note : pos > 0, otherwise, impossible to finish reading last block */
                 zds->hostageByte=1;
             }
             return 1;
-        }
+        }  /* nextSrcSizeHint==0 */
         nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block);   /* preload header of next block */
         if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC);   /* should never happen */
         nextSrcSizeHint -= zds->inPos;   /* already loaded*/
diff --git a/lib/zstd.h b/lib/zstd.h
index 81f9eb737..bb043e486 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -529,7 +529,7 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
 
 
-/* !!! Soon to be deprecated !!! */
+/* !!! To be deprecated !!! */
 typedef enum {
     ZSTD_p_forceWindow,   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
     ZSTD_p_forceRawDict   /* Force loading dictionary in "content-only" mode (no header analysis) */
@@ -815,6 +815,22 @@ ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
  *  Create a ZSTD decompression context using external alloc and free functions */
 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
 
+/*! ZSTD_initStaticDCtx() : initialize a fixed-size zstd decompression context
+ *  workspace: The memory area to emplace the context into.
+ *             Provided pointer must 8-bytes aligned.
+ *             It must outlive context usage.
+ *  workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
+ *                 to determine how large workspace must be to support scenario.
+ * @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
+ *  Note : zstd will never resize nor malloc() when using a static dctx.
+ *         If it needs more memory than available, it will simply error out.
+ *  Note 2 : there is no corresponding "free" function.
+ *           Since workspace was allocated externally, it must be freed externally.
+ *  Limitation : currently not compatible with internal DDict creation,
+ *               such as ZSTD_initDStream_usingDict().
+ */
+ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
+
 /*! ZSTD_createDDict_byReference() :
  *  Create a digested dictionary, ready to start decompression operation without startup delay.
  *  Dictionary content is simply referenced, and therefore stays in dictBuffer.
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index a1dc6ba77..b2de6ec69 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -74,7 +74,6 @@ static clock_t FUZ_clockSpan(clock_t cStart)
     return clock() - cStart;   /* works even when overflow; max span ~ 30mn */
 }
 
-
 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
 static unsigned FUZ_rand(unsigned* src)
 {
@@ -104,6 +103,7 @@ static unsigned FUZ_highbit32(U32 v32)
 #define CHECK_V(var, fn)  size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
 #define CHECK(fn)  { CHECK_V(err, fn); }
 #define CHECKPLUS(var, fn, more)  { CHECK_V(var, fn); more; }
+
 static int basicUnitTests(U32 seed, double compressibility)
 {
     size_t const CNBuffSize = 5 MB;
@@ -195,14 +195,19 @@ static int basicUnitTests(U32 seed, double compressibility)
     DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(STATIC_CCTX_LEVEL, 0, 0);
         size_t const staticCCtxSize = ZSTD_estimateCStreamSize(cParams);
-        void* staticCCtxBuffer = malloc(staticCCtxSize);
-        if (staticCCtxBuffer==NULL) {
+        void* const staticCCtxBuffer = malloc(staticCCtxSize);
+        size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
+        void* const staticDCtxBuffer = malloc(staticDCtxSize);
+        if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
+            free(staticCCtxBuffer);
+            free(staticDCtxBuffer);
             DISPLAY("Not enough memory, aborting\n");
             testResult = 1;
             goto _end;
         }
         {   ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
-            if (staticCCtx==NULL) goto _output_error;
+            ZSTD_DCtx* staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
+            if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
             DISPLAYLEVEL(4, "OK \n");
 
             DISPLAYLEVEL(4, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
@@ -215,13 +220,24 @@ static int basicUnitTests(U32 seed, double compressibility)
                             compressedBuffer, ZSTD_compressBound(CNBuffSize),
                             CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL),
                       cSize=r );
-            DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+            DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n",
+                            (U32)cSize, (double)cSize/CNBuffSize*100);
 
-            DISPLAYLEVEL(4, "test%3i : decompress verification test : ", testNb++);
-            { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+            DISPLAYLEVEL(4, "test%3i : simple decompression test with static DCtx : ", testNb++);
+            { size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                decodedBuffer, CNBuffSize,
+                                                compressedBuffer, cSize);
               if (r != CNBuffSize) goto _output_error; }
             DISPLAYLEVEL(4, "OK \n");
 
+            DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+            {   size_t u;
+                for (u=0; u
Date: Thu, 25 May 2017 15:07:37 -0700
Subject: [PATCH 027/109] updated ZSTD_estimate?DictSize() to pass parameter
 byReference

resulting ?Dict object is smaller when created byReference.
Seems better than a documentation note.
---
 doc/zstd_manual.html             | 21 ++++++++++++++++++---
 lib/compress/zstd_compress.c     |  5 +++--
 lib/decompress/zstd_decompress.c |  6 +++---
 lib/zstd.h                       | 26 +++++++++++++++++++++-----
 tests/fuzzer.c                   |  7 ++++---
 tests/zstreamtest.c              |  6 ++++--
 6 files changed, 53 insertions(+), 18 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 72da13650..f6a09ef0c 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -412,9 +412,9 @@ size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
          In this case, get additional size by using ZSTD_estimate?DictSize 
 


 
-
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
-size_t ZSTD_estimateDDictSize(size_t dictSize);
-
  Note : if dictionary is created "byReference", reduce estimation by dictSize 
+
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
+size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
+
  Note : dictionary created "byReference" are smaller 
 


 
 
Advanced compression functions

@@ -718,6 +718,21 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
 
  Create a ZSTD_DDict using external alloc and free, optionally by reference 
 


 
+
ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
+                                 const void* dict, size_t dictSize,
+                                 unsigned byReference);
+
  Generate a digested dictionary in provided memory area.
+  workspace: The memory area to emplace the dictionary into.
+             Provided pointer must 8-bytes aligned.
+             It must outlive dictionary usage.
+  workspaceSize: Use ZSTD_estimateDDictSize()
+                 to determine how large workspace must be.
+ @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
+  Note : there is no corresponding "free" function.
+         Since workspace was allocated externally, it must be freed externally.
+ 
+


+
 
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
 
  Provides the dictID stored within dictionary.
   if @return == 0, the dictionary is not conformant with Zstandard specification.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 58ed98e9b..aa9f64c5c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3237,10 +3237,11 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
 
 /*! ZSTD_estimateCDictSize() :
  *  Estimate amount of memory that will be needed to create a dictionary with following arguments */
-size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
+size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
 {
     cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
-    return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams);
+    return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize(cParams)
+           + (byReference ? 0 : dictSize);
 }
 
 size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 2db493e51..8b1381184 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2053,10 +2053,10 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
 
 /*! ZSTD_estimateDDictSize() :
  *  Estimate amount of memory that will be needed to create a dictionary for decompression.
- *  Note : if dictionary is created "byReference", reduce this amount by dictSize */
-size_t ZSTD_estimateDDictSize(size_t dictSize)
+ *  Note : dictionary created "byReference" are smaller */
+size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference)
 {
-    return dictSize + sizeof(ZSTD_DDict);
+    return sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
 }
 
 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
diff --git a/lib/zstd.h b/lib/zstd.h
index bb043e486..de3b2c30d 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -345,8 +345,6 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 #endif  /* ZSTD_H_235446 */
 
 
-#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
-#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
 
 /****************************************************************************************
  * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
@@ -356,6 +354,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
  * Use them only in association with static linking.
  * ***************************************************************************************/
 
+#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
+#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
+
 /* --- Constants ---*/
 #define ZSTD_MAGICNUMBER            0xFD2FB528   /* >= v0.8.0 */
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
@@ -500,9 +501,9 @@ ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
 ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
 /*! ZSTD_estimate?DictSize() :
- *  Note : if dictionary is created "byReference", reduce estimation by dictSize */
-ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize);
-ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
+ *  Note : dictionary created "byReference" are smaller */
+ZSTDLIB_API size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
+ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
 
 
 /***************************************
@@ -842,6 +843,21 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, siz
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
                                                   unsigned byReference, ZSTD_customMem customMem);
 
+/*! ZSTD_initStaticDDict() :
+ *  Generate a digested dictionary in provided memory area.
+ *  workspace: The memory area to emplace the dictionary into.
+ *             Provided pointer must 8-bytes aligned.
+ *             It must outlive dictionary usage.
+ *  workspaceSize: Use ZSTD_estimateDDictSize()
+ *                 to determine how large workspace must be.
+ * @return : pointer to ZSTD_DDict*, or NULL if error (size too small)
+ *  Note : there is no corresponding "free" function.
+ *         Since workspace was allocated externally, it must be freed externally.
+ */
+ZSTDLIB_API ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
+                                             const void* dict, size_t dictSize,
+                                             unsigned byReference);
+
 /*! ZSTD_getDictID_fromDict() :
  *  Provides the dictID stored within dictionary.
  *  if @return == 0, the dictionary is not conformant with Zstandard specification.
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index b2de6ec69..83b95974f 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -490,14 +490,15 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
-            size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize);
+            size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize, 1);
             DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
         }
 
         DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
-            ZSTD_customMem customMem = { NULL, NULL, NULL };
-            ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
+            ZSTD_customMem const customMem = { NULL, NULL, NULL };
+            ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
+                                    1 /* by Referece */, cParams, customMem);
             cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
                                                  CNBuffer, CNBuffSize, cdict);
             ZSTD_freeCDict(cdict);
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index e9e17031c..0f73b8cd4 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -209,7 +209,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
         size_t const s = ZSTD_estimateCStreamSize(cParams)
-                       + ZSTD_estimateCDictSize(cParams, dictSize);  /* uses ZSTD_initCStream_usingDict() */
+                        /* uses ZSTD_initCStream_usingDict() */
+                       + ZSTD_estimateCDictSize(cParams, dictSize, 0);
             if (ZSTD_isError(s)) goto _output_error;
             DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
     }
@@ -281,7 +282,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         if (gfhError!=0) goto _output_error;
         DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
         {   size_t const s = ZSTD_estimateDStreamSize(fhi)
-                           + ZSTD_estimateDDictSize(dictSize);  /* uses ZSTD_initDStream_usingDict() */
+                            /* uses ZSTD_initDStream_usingDict() */
+                           + ZSTD_estimateDDictSize(dictSize, 0);
             if (ZSTD_isError(s)) goto _output_error;
             DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
     }   }

From 57827f906f63eee0afaa51b2a82f6d2967f6f138 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 25 May 2017 15:44:06 -0700
Subject: [PATCH 028/109] added ZSTD_initStaticDDict()

---
 doc/zstd_manual.html             |  5 +--
 lib/decompress/zstd_decompress.c | 59 ++++++++++++++++++++++----------
 lib/zstd.h                       |  5 +--
 tests/fuzzer.c                   | 13 ++++++-
 4 files changed, 59 insertions(+), 23 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index f6a09ef0c..5040dd7d3 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -709,8 +709,9 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
 
 
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
 
  Create a digested dictionary, ready to start decompression operation without startup delay.
-  Dictionary content is simply referenced, and therefore stays in dictBuffer.
-  It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict 
+  Dictionary content is referenced, and therefore stays in dictBuffer.
+  It is important that dictBuffer outlives DDict,
+  it must remain read accessible throughout the lifetime of DDict 
 


 
 
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 8b1381184..e3721929e 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1987,6 +1987,27 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
 }
 
 
+static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_t dictSize, unsigned byReference)
+{
+    if ((byReference) || (!dict) || (!dictSize)) {
+        ddict->dictBuffer = NULL;
+        ddict->dictContent = dict;
+    } else {
+        void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
+        if (!internalBuffer) return ERROR(memory_allocation);
+        memcpy(internalBuffer, dict, dictSize);
+        ddict->dictBuffer = internalBuffer;
+        ddict->dictContent = internalBuffer;
+    }
+    ddict->dictSize = dictSize;
+    ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
+
+    /* parse dictionary content */
+    CHECK_F( ZSTD_loadEntropy_inDDict(ddict) );
+
+    return 0;
+}
+
 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
 {
     if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
@@ -1996,24 +2017,10 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigne
         if (!ddict) return NULL;
         ddict->cMem = customMem;
 
-        if ((byReference) || (!dict) || (!dictSize)) {
-            ddict->dictBuffer = NULL;
-            ddict->dictContent = dict;
-        } else {
-            void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
-            if (!internalBuffer) { ZSTD_freeDDict(ddict); return NULL; }
-            memcpy(internalBuffer, dict, dictSize);
-            ddict->dictBuffer = internalBuffer;
-            ddict->dictContent = internalBuffer;
+        if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, byReference) )) {
+            ZSTD_freeDDict(ddict);
+            return NULL;
         }
-        ddict->dictSize = dictSize;
-        ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
-        /* parse dictionary content */
-        {   size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
-            if (ZSTD_isError(errorCode)) {
-                ZSTD_freeDDict(ddict);
-                return NULL;
-        }   }
 
         return ddict;
     }
@@ -2029,7 +2036,6 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
     return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
 }
 
-
 /*! ZSTD_createDDict_byReference() :
  *  Create a digested dictionary, to start decompression without startup delay.
  *  Dictionary content is simply referenced, it will be accessed during decompression.
@@ -2041,6 +2047,23 @@ ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize
 }
 
 
+ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
+                                 const void* dict, size_t dictSize,
+                                 unsigned byReference)
+{
+    size_t const neededSpace = sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
+    ZSTD_DDict* const ddict = (ZSTD_DDict*)workspace;
+    if (workspaceSize < neededSpace) return NULL;  /* minimum size */
+    if (!byReference) {
+        memcpy(ddict+1, dict, dictSize);  /* local copy */
+        dict = ddict+1;
+    }
+    if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, 1 /* byRef */) ))
+        return NULL;
+    return ddict;
+}
+
+
 size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;   /* support free on NULL */
diff --git a/lib/zstd.h b/lib/zstd.h
index de3b2c30d..82501994e 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -834,8 +834,9 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize
 
 /*! ZSTD_createDDict_byReference() :
  *  Create a digested dictionary, ready to start decompression operation without startup delay.
- *  Dictionary content is simply referenced, and therefore stays in dictBuffer.
- *  It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
+ *  Dictionary content is referenced, and therefore stays in dictBuffer.
+ *  It is important that dictBuffer outlives DDict,
+ *  it must remain read accessible throughout the lifetime of DDict */
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
 
 /*! ZSTD_createDDict_advanced() :
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 83b95974f..90cfe648b 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -405,13 +405,24 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "OK \n");
 
         DISPLAYLEVEL(4, "test%3i : decompress with DDict : ", testNb++);
-        {   ZSTD_DDict* const ddict = ZSTD_createDDict_byReference(CNBuffer, dictSize);
+        {   ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
             size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
             if (r != CNBuffSize - dictSize) goto _output_error;
             DISPLAYLEVEL(4, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict));
             ZSTD_freeDDict(ddict);
         }
 
+        DISPLAYLEVEL(4, "test%3i : decompress with static DDict : ", testNb++);
+        {   size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, 0);
+            void* ddictBuffer = malloc(ddictBufferSize);
+            if (ddictBuffer == NULL) goto _output_error;
+            {   ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, 0);
+                size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
+                if (r != CNBuffSize - dictSize) goto _output_error;
+            }
+            DISPLAYLEVEL(4, "OK (size of static DDict : %u) \n", (U32)ddictBufferSize);
+        }
+
         DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
         {   size_t const testSize = CNBuffSize / 3;
             {   ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize);

From cdf7e82222758cecb01438f09cf1f7b08bf1688d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 25 May 2017 18:05:49 -0700
Subject: [PATCH 029/109] Added ZSTD_initStaticCDict()

---
 doc/zstd_manual.html             |  22 +++++-
 lib/compress/zstd_compress.c     | 111 ++++++++++++++++++++++---------
 lib/decompress/zstd_decompress.c |   3 +-
 lib/zstd.h                       |  23 ++++++-
 tests/fuzzer.c                   |  23 +++++++
 5 files changed, 145 insertions(+), 37 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 5040dd7d3..a4b59bc46 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -408,8 +408,8 @@ size_t ZSTD_estimateDCtxSize(void);
 
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
 size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
-         an internal ?Dict will be created, which size is not estimated.
-         In this case, get additional size by using ZSTD_estimate?DictSize 
+         an internal ?Dict will be created, which size is not estimated here.
+         In this case, get total size by adding ZSTD_estimate?DictSize 
 


 
 
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
@@ -460,6 +460,24 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
 
  Create a ZSTD_CDict using external alloc and free, and customized compression parameters 
 


 
+
ZSTD_CDict* ZSTD_initStaticCDict(
+                void* workspace, size_t workspaceSize,
+          const void* dict, size_t dictSize, unsigned byReference,
+                ZSTD_compressionParameters cParams);
+
  Generate a digested dictionary in provided memory area.
+  workspace: The memory area to emplace the dictionary into.
+             Provided pointer must 8-bytes aligned.
+             It must outlive dictionary usage.
+  workspaceSize: Use ZSTD_estimateCDictSize()
+                 to determine how large workspace must be.
+  cParams : use ZSTD_getCParams() to transform a compression level
+            into its relevants cParams.
+ @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+  Note : there is no corresponding "free" function.
+         Since workspace was allocated externally, it must be freed externally.
+ 
+


+
 
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
 
   @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
    `estimatedSrcSize` value is optional, select 0 if not known 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index aa9f64c5c..0e93abccb 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3239,7 +3239,6 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
  *  Estimate amount of memory that will be needed to create a dictionary with following arguments */
 size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
 {
-    cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
     return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize(cParams)
            + (byReference ? 0 : dictSize);
 }
@@ -3258,6 +3257,34 @@ static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_
     return params;
 }
 
+static size_t ZSTD_initCDict_internal(
+                    ZSTD_CDict* cdict,
+              const void* dictBuffer, size_t dictSize, unsigned byReference,
+                    ZSTD_compressionParameters cParams)
+{
+    if ((byReference) || (!dictBuffer) || (!dictSize)) {
+        cdict->dictBuffer = NULL;
+        cdict->dictContent = dictBuffer;
+    } else {
+        void* const internalBuffer = ZSTD_malloc(dictSize, cdict->refContext->customMem);
+        if (!internalBuffer) return ERROR(memory_allocation);
+        memcpy(internalBuffer, dictBuffer, dictSize);
+        cdict->dictBuffer = internalBuffer;
+        cdict->dictContent = internalBuffer;
+    }
+    cdict->dictContentSize = dictSize;
+
+    {   ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */,
+                    0 /* checksumFlag */, 0 /* noDictIDFlag */ };  /* dummy */
+        ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams);
+        CHECK_F( ZSTD_compressBegin_advanced(cdict->refContext,
+                                            cdict->dictContent, dictSize,
+                                            params, 0 /* srcSize */) );
+    }
+
+    return 0;
+}
+
 ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
                                       ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
 {
@@ -3269,44 +3296,19 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
         ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
 
         if (!cdict || !cctx) {
-            DEBUGLOG(5, "!cdict || !cctx");
             ZSTD_free(cdict, customMem);
             ZSTD_freeCCtx(cctx);
             return NULL;
         }
+        cdict->refContext = cctx;
 
-        if ((byReference) || (!dictBuffer) || (!dictSize)) {
-            cdict->dictBuffer = NULL;
-            cdict->dictContent = dictBuffer;
-        } else {
-            void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
-            if (!internalBuffer) {
-                DEBUGLOG(5, "!internalBuffer");
-                ZSTD_free(cctx, customMem);
-                ZSTD_free(cdict, customMem);
-                return NULL;
-            }
-            memcpy(internalBuffer, dictBuffer, dictSize);
-            cdict->dictBuffer = internalBuffer;
-            cdict->dictContent = internalBuffer;
+        if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
+                                        dictBuffer, dictSize, byReference,
+                                        cParams) )) {
+            ZSTD_freeCDict(cdict);
+            return NULL;
         }
 
-        {   ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */,
-                    0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */
-            ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams);
-            size_t const errorCode = ZSTD_compressBegin_advanced(cctx,
-                                cdict->dictContent, dictSize, params, 0);
-            if (ZSTD_isError(errorCode)) {
-                DEBUGLOG(5, "ZSTD_compressBegin_advanced error : %s",
-                            ZSTD_getErrorName(errorCode));
-                ZSTD_free(cdict->dictBuffer, customMem);
-                ZSTD_free(cdict, customMem);
-                ZSTD_freeCCtx(cctx);
-                return NULL;
-        }   }
-
-        cdict->refContext = cctx;
-        cdict->dictContentSize = dictSize;
         return cdict;
     }
 }
@@ -3336,6 +3338,51 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
     }
 }
 
+/*! ZSTD_initStaticCDict_advanced() :
+ *  Generate a digested dictionary in provided memory area.
+ *  workspace: The memory area to emplace the dictionary into.
+ *             Provided pointer must 8-bytes aligned.
+ *             It must outlive dictionary usage.
+ *  workspaceSize: Use ZSTD_estimateCDictSize()
+ *                 to determine how large workspace must be.
+ *  cParams : use ZSTD_getCParams() to transform a compression level
+ *            into its relevants cParams.
+ * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+ *  Note : there is no corresponding "free" function.
+ *         Since workspace was allocated externally, it must be freed externally.
+ */
+ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
+                           const void* dict, size_t dictSize, unsigned byReference,
+                                 ZSTD_compressionParameters cParams)
+{
+    size_t const cctxSize = ZSTD_estimateCCtxSize(cParams);
+    size_t const neededSize = sizeof(ZSTD_CDict) + (byReference ? 0 : dictSize)
+                            + cctxSize;
+    ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
+    void* ptr;
+    DEBUGLOG(2, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
+    if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
+    DEBUGLOG(2, "(workspaceSize < neededSize) : (%u < %u) => %u",
+        (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
+    if (workspaceSize < neededSize) return NULL;
+
+    if (!byReference) {
+        memcpy(cdict+1, dict, dictSize);
+        dict = cdict+1;
+        ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize;
+    } else {
+        ptr = cdict+1;
+    }
+    cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize);
+
+    if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
+                                        dict, dictSize, 1 /* by Reference */,
+                                        cParams) ))
+        return NULL;
+
+    return cdict;
+}
+
 static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
     return ZSTD_getParamsFromCCtx(cdict->refContext);
 }
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index e3721929e..fe7446ab2 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2053,7 +2053,8 @@ ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
 {
     size_t const neededSpace = sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
     ZSTD_DDict* const ddict = (ZSTD_DDict*)workspace;
-    if (workspaceSize < neededSpace) return NULL;  /* minimum size */
+    if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
+    if (workspaceSize < neededSpace) return NULL;
     if (!byReference) {
         memcpy(ddict+1, dict, dictSize);  /* local copy */
         dict = ddict+1;
diff --git a/lib/zstd.h b/lib/zstd.h
index 82501994e..b1d8ecadd 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -495,8 +495,8 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
 
 /*! ZSTD_estimate?StreamSize() :
  *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
- *         an internal ?Dict will be created, which size is not estimated.
- *         In this case, get additional size by using ZSTD_estimate?DictSize */
+ *         an internal ?Dict will be created, which size is not estimated here.
+ *         In this case, get total size by adding ZSTD_estimate?DictSize */
 ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
 ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
@@ -540,6 +540,7 @@ typedef enum {
  *  @result : 0, or an error code (which can be tested with ZSTD_isError()) */
 ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
 
+
 /*! ZSTD_createCDict_byReference() :
  *  Create a digested dictionary for compression
  *  Dictionary content is simply referenced, and therefore stays in dictBuffer.
@@ -551,6 +552,24 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
                                                   ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
 
+/*! ZSTD_initStaticCDict_advanced() :
+ *  Generate a digested dictionary in provided memory area.
+ *  workspace: The memory area to emplace the dictionary into.
+ *             Provided pointer must 8-bytes aligned.
+ *             It must outlive dictionary usage.
+ *  workspaceSize: Use ZSTD_estimateCDictSize()
+ *                 to determine how large workspace must be.
+ *  cParams : use ZSTD_getCParams() to transform a compression level
+ *            into its relevants cParams.
+ * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+ *  Note : there is no corresponding "free" function.
+ *         Since workspace was allocated externally, it must be freed externally.
+ */
+ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict(
+                            void* workspace, size_t workspaceSize,
+                      const void* dict, size_t dictSize, unsigned byReference,
+                            ZSTD_compressionParameters cParams);
+
 /*! ZSTD_getCParams() :
 *   @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
 *   `estimatedSrcSize` value is optional, select 0 if not known */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 90cfe648b..ee0aff08c 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -531,6 +531,29 @@ static int basicUnitTests(U32 seed, double compressibility)
                   if (r != CNBuffSize) goto _output_error);
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : compress with static CDict : ", testNb++);
+        {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
+            size_t const cdictSize = ZSTD_estimateCDictSize(cParams, dictSize, 0);
+            void* const cdictBuffer = malloc(cdictSize);
+            if (cdictBuffer==NULL) goto _output_error;
+            {   ZSTD_CDict* const cdict = ZSTD_initStaticCDict(cdictBuffer, cdictSize,
+                                            dictBuffer, dictSize, 0 /* by Reference */,
+                                            cParams);
+                if (cdict == NULL) {
+                    DISPLAY("ZSTD_initStaticCDict failed ");
+                    goto _output_error;
+                }
+                cSize = ZSTD_compress_usingCDict(cctx,
+                                compressedBuffer, ZSTD_compressBound(CNBuffSize),
+                                CNBuffer, CNBuffSize, cdict);
+                if (ZSTD_isError(cSize)) {
+                    DISPLAY("ZSTD_compress_usingCDict failed ");
+                    goto _output_error;
+            }   }
+            free(cdictBuffer);
+        }
+        DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
         DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
         {   ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
             ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);

From 7028cbd7fdf153feb795251af9cae20bbbe22975 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 25 May 2017 18:29:08 -0700
Subject: [PATCH 030/109] fixed a few code comments : ZSTD_getFrameParams =>
 ZSTD_getFrameHeader

---
 doc/zstd_manual.html | 8 ++++----
 lib/zstd.h           | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index a4b59bc46..4d5df5977 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -93,7 +93,7 @@
    note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
             Always ensure result fits within application's authorized limits.
             Each application can set its own limits.
-   note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. 
+   note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameHeader() to know more. 
 


 
 
Helper functions
int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
@@ -773,7 +773,7 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
     Note : this use case also happens when using a non-conformant dictionary.
   - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
   - This is not a Zstandard frame.
-  When identifying the exact failure cause, it's possible to use ZSTD_getFrameParams(), which will provide a more precise error code. 
+  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. 
 


 
 
Advanced streaming functions

@@ -850,7 +850,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
 
-  First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
+  First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
   It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
   such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
   and the dictionary ID used.
@@ -897,7 +897,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
   b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
   c) Frame Content - any content (User Data) of length equal to Frame Size
   For skippable frames ZSTD_decompressContinue() always returns 0.
-  For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+  For skippable frames ZSTD_getFrameHeader() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
     Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
            For purposes of decompression, it is valid in both cases to skip the frame using
            ZSTD_findFrameCompressedSize to find its size in bytes.
diff --git a/lib/zstd.h b/lib/zstd.h
index b1d8ecadd..bc3d7a0e9 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -112,7 +112,7 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
  *   note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
  *            Always ensure result fits within application's authorized limits.
  *            Each application can set its own limits.
- *   note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
+ *   note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameHeader() to know more. */
 ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
 
 
@@ -899,7 +899,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
  *    Note : this use case also happens when using a non-conformant dictionary.
  *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
  *  - This is not a Zstandard frame.
- *  When identifying the exact failure cause, it's possible to use ZSTD_getFrameParams(), which will provide a more precise error code. */
+ *  When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
 ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
 
 
@@ -994,7 +994,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
 
-  First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams().
+  First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
   It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame,
   such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
   and the dictionary ID used.
@@ -1041,7 +1041,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
   c) Frame Content - any content (User Data) of length equal to Frame Size
   For skippable frames ZSTD_decompressContinue() always returns 0.
-  For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+  For skippable frames ZSTD_getFrameHeader() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
     Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
            For purposes of decompression, it is valid in both cases to skip the frame using
            ZSTD_findFrameCompressedSize to find its size in bytes.

From b8136f019a7dc50b51fc51e68cfd38bace101057 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 27 May 2017 00:03:08 -0700
Subject: [PATCH 031/109] static dctx is incompatible with legacy support

documented, and runtime tested
---
 doc/zstd_manual.html             | 3 ++-
 lib/decompress/zstd_decompress.c | 9 ++++++++-
 lib/zstd.h                       | 3 ++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index a4b59bc46..156e7dd92 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -718,7 +718,8 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
  @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
   Note : zstd will never resize nor malloc() when using a static dctx.
          If it needs more memory than available, it will simply error out.
-  Note 2 : there is no corresponding "free" function.
+  Note 2 : static dctx is incompatible with legacy support
+  Note 3 : there is no corresponding "free" function.
            Since workspace was allocated externally, it must be freed externally.
   Limitation : currently not compatible with internal DDict creation,
                such as ZSTD_initDStream_usingDict().
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index fe7446ab2..a59724386 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1586,6 +1586,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
             size_t decodedSize;
             size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
             if (ZSTD_isError(frameSize)) return frameSize;
+            /* legacy support is incompatible with static dctx */
+            if (dctx->staticSize) return ERROR(memory_allocation);
 
             decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
 
@@ -2258,8 +2260,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     U32 someMoreWork = 1;
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
-    if (zds->legacyVersion)
+    if (zds->legacyVersion) {
+        /* legacy support is incompatible with static dctx */
+        if (zds->staticSize) return ERROR(memory_allocation);
         return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
+    }
 #endif
 
     while (someMoreWork) {
@@ -2277,6 +2282,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     if (legacyVersion) {
                         const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
                         size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
+                        /* legacy support is incompatible with static dctx */
+                        if (zds->staticSize) return ERROR(memory_allocation);
                         CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
                                                        dict, dictSize));
                         zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
diff --git a/lib/zstd.h b/lib/zstd.h
index b1d8ecadd..3bbdb76f5 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -844,7 +844,8 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
  * @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
  *  Note : zstd will never resize nor malloc() when using a static dctx.
  *         If it needs more memory than available, it will simply error out.
- *  Note 2 : there is no corresponding "free" function.
+ *  Note 2 : static dctx is incompatible with legacy support
+ *  Note 3 : there is no corresponding "free" function.
  *           Since workspace was allocated externally, it must be freed externally.
  *  Limitation : currently not compatible with internal DDict creation,
  *               such as ZSTD_initDStream_usingDict().

From e07115910137fda805c1ef14f7834ca7ba896677 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 27 May 2017 00:21:33 -0700
Subject: [PATCH 032/109] mtctx->jobs allocate its own memory space

to make ZSTDMT_CCtx_s size predictable
so that it can be included in CCtx
---
 lib/compress/zstdmt_compress.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index fc7f52a29..d41825e80 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -271,6 +271,7 @@ _endJob:
 
 struct ZSTDMT_CCtx_s {
     POOL_ctx* factory;
+    ZSTDMT_jobDescription* jobs;
     ZSTDMT_bufferPool* buffPool;
     ZSTDMT_CCtxPool* cctxPool;
     pthread_mutex_t jobCompleted_mutex;
@@ -294,10 +295,9 @@ struct ZSTDMT_CCtx_s {
     size_t sectionSize;
     ZSTD_CDict* cdict;
     ZSTD_CStream* cstream;
-    ZSTDMT_jobDescription jobs[1];   /* variable size (must lies at the end) */
 };
 
-ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
+ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads)
 {
     ZSTDMT_CCtx* cctx;
     U32 const minNbJobs = nbThreads + 2;
@@ -306,7 +306,7 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
     DEBUGLOG(5, "nbThreads : %u  ; minNbJobs : %u ;  nbJobsLog2 : %u ;  nbJobs : %u  \n",
             nbThreads, minNbJobs, nbJobsLog2, nbJobs);
     if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
-    cctx = (ZSTDMT_CCtx*) calloc(1, sizeof(ZSTDMT_CCtx) + nbJobs*sizeof(ZSTDMT_jobDescription));
+    cctx = (ZSTDMT_CCtx*) calloc(1, sizeof(ZSTDMT_CCtx));
     if (!cctx) return NULL;
     cctx->nbThreads = nbThreads;
     cctx->jobIDMask = nbJobs - 1;
@@ -314,9 +314,10 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads)
     cctx->sectionSize = 0;
     cctx->overlapRLog = 3;
     cctx->factory = POOL_create(nbThreads, 1);
+    cctx->jobs = malloc(nbJobs * sizeof(*cctx->jobs));
     cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
     cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
-    if (!cctx->factory | !cctx->buffPool | !cctx->cctxPool) {  /* one object was not created */
+    if (!cctx->factory | !cctx->jobs | !cctx->buffPool | !cctx->cctxPool) {
         ZSTDMT_freeCCtx(cctx);
         return NULL;
     }
@@ -356,6 +357,7 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
     POOL_free(mtctx->factory);
     if (!mtctx->allJobsCompleted) ZSTDMT_releaseAllJobResources(mtctx); /* stop workers first */
     ZSTDMT_freeBufferPool(mtctx->buffPool);  /* release job resources into pools first */
+    free(mtctx->jobs);
     ZSTDMT_freeCCtxPool(mtctx->cctxPool);
     ZSTD_freeCDict(mtctx->cdict);
     ZSTD_freeCStream(mtctx->cstream);
@@ -491,7 +493,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
 /* =======      Streaming API     ======= */
 /* ====================================== */
 
-static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) {
+static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
+{
     while (zcs->doneJobID < zcs->nextJobID) {
         unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
         PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);

From b6dec4c3ae18786d6183274b3bcb5fe7387ea2bc Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 27 May 2017 17:09:06 -0700
Subject: [PATCH 033/109] fixed minor cast warning

---
 lib/compress/zstdmt_compress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index d41825e80..0c6bc7249 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -314,7 +314,7 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads)
     cctx->sectionSize = 0;
     cctx->overlapRLog = 3;
     cctx->factory = POOL_create(nbThreads, 1);
-    cctx->jobs = malloc(nbJobs * sizeof(*cctx->jobs));
+    cctx->jobs = (ZSTDMT_jobDescription*) malloc(nbJobs * sizeof(*cctx->jobs));
     cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
     cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
     if (!cctx->factory | !cctx->jobs | !cctx->buffPool | !cctx->cctxPool) {

From 44e45e8423f1a8590009456dc48cda1164d90aed Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 30 May 2017 16:12:06 -0700
Subject: [PATCH 034/109] added ZSTDMT_createCCtx_advanced()

make it possible to use custom allocators
---
 doc/zstd_manual.html           |   2 +
 lib/common/zstd_common.c       |  27 +++++-
 lib/common/zstd_internal.h     |   1 +
 lib/compress/zstdmt_compress.c | 151 +++++++++++++++++++--------------
 lib/compress/zstdmt_compress.h |   1 +
 lib/zstd.h                     |   2 +
 6 files changed, 118 insertions(+), 66 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 360a6e77a..2c66409cb 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -338,6 +338,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
Custom memory allocation functions
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+/* use this constant to defer to stdlib's functions */
+static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL};
 


 
Frame size functions

 
diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 5e207e7d1..9b46df1d8 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -12,7 +12,8 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
-#include          /* malloc */
+#include       /* malloc, calloc, free */
+#include       /* memset */
 #include "error_private.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
@@ -65,11 +66,29 @@ void ZSTD_defaultFreeFunction(void* opaque, void* address)
 
 void* ZSTD_malloc(size_t size, ZSTD_customMem customMem)
 {
-    return customMem.customAlloc(customMem.opaque, size);
+    if (customMem.customAlloc)
+        return customMem.customAlloc(customMem.opaque, size);
+    return malloc(size);
+}
+
+void* ZSTD_calloc(size_t size, ZSTD_customMem customMem)
+{
+    if (customMem.customAlloc) {
+        /* calloc implemented as malloc+memset;
+         * not as efficient, but our best guess for custom malloc */
+        void* const ptr = customMem.customAlloc(customMem.opaque, size);
+        memset(ptr, 0, size);
+        return ptr;
+    }
+    return calloc(1, size);
 }
 
 void ZSTD_free(void* ptr, ZSTD_customMem customMem)
 {
-    if (ptr!=NULL)
-        customMem.customFree(customMem.opaque, ptr);
+    if (ptr!=NULL) {
+        if (customMem.customFree)
+            customMem.customFree(customMem.opaque, ptr);
+        else
+            free(ptr);
+    }
 }
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 2533333ba..5ab4aaeff 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -244,6 +244,7 @@ void ZSTD_defaultFreeFunction(void* opaque, void* address);
 static const ZSTD_customMem defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
 #endif
 void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
+void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);
 void ZSTD_free(void* ptr, ZSTD_customMem customMem);
 
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 0c6bc7249..bd3de4586 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -19,29 +19,33 @@
 
 
 /* ======   Dependencies   ====== */
-#include    /* malloc */
-#include    /* memcpy */
-#include "pool.h"     /* threadpool */
+#include      /* memcpy, memset */
+#include "pool.h"       /* threadpool */
 #include "threading.h"  /* mutex */
-#include "zstd_internal.h"   /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
+#include "zstd_internal.h"  /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
 #include "zstdmt_compress.h"
 
 
 /* ======   Debug   ====== */
-#if 0
+#if defined(ZSTDMT_DEBUG)
 
 #  include 
 #  include 
 #  include 
-   static unsigned g_debugLevel = 5;
-#  define DEBUGLOGRAW(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __VA_ARGS__); }
-#  define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); }
+#  define DEBUGLOGRAW(l, ...) if (l<=ZSTDMT_DEBUG) { fprintf(stderr, __VA_ARGS__); }
+#  define DEBUGLOG(l, ...) {            \
+    if (l<=ZSTDMT_DEBUG) {              \
+        fprintf(stderr, __FILE__ ": "); \
+        fprintf(stderr, __VA_ARGS__);   \
+        fprintf(stderr, " \n");         \
+    }                                   \
+}
 
-#  define DEBUG_PRINTHEX(l,p,n) { \
-    unsigned debug_u;                   \
-    for (debug_u=0; debug_u<(n); debug_u++)           \
+#  define DEBUG_PRINTHEX(l,p,n) {            \
+    unsigned debug_u;                        \
+    for (debug_u=0; debug_u<(n); debug_u++)  \
         DEBUGLOGRAW(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \
-    DEBUGLOGRAW(l, " \n");       \
+    DEBUGLOGRAW(l, " \n");                   \
 }
 
 static unsigned long long GetCurrentClockTimeMicroseconds(void)
@@ -54,17 +58,18 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
 }
 
 #define MUTEX_WAIT_TIME_DLEVEL 5
-#define PTHREAD_MUTEX_LOCK(mutex) \
-if (g_debugLevel>=MUTEX_WAIT_TIME_DLEVEL) { \
-    unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \
-    pthread_mutex_lock(mutex); \
-    {   unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \
-        unsigned long long const elapsedTime = (afterTime-beforeTime); \
-        if (elapsedTime > 1000) {  /* or whatever threshold you like; I'm using 1 millisecond here */ \
-            DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
-               elapsedTime, #mutex); \
-    }   } \
-} else pthread_mutex_lock(mutex);
+#define PTHREAD_MUTEX_LOCK(mutex) {               \
+    if (ZSTDMT_DEBUG>=MUTEX_WAIT_TIME_DLEVEL) {   \
+        unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \
+        pthread_mutex_lock(mutex);                \
+        {   unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \
+            unsigned long long const elapsedTime = (afterTime-beforeTime); \
+            if (elapsedTime > 1000) {  /* or whatever threshold you like; I'm using 1 millisecond here */ \
+                DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
+                   elapsedTime, #mutex);          \
+        }   }                                     \
+    } else pthread_mutex_lock(mutex);             \
+}
 
 #else
 
@@ -87,16 +92,19 @@ static const buffer_t g_nullBuffer = { NULL, 0 };
 typedef struct ZSTDMT_bufferPool_s {
     unsigned totalBuffers;
     unsigned nbBuffers;
+    ZSTD_customMem cMem;
     buffer_t bTable[1];   /* variable size */
 } ZSTDMT_bufferPool;
 
-static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads)
+static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_customMem cMem)
 {
     unsigned const maxNbBuffers = 2*nbThreads + 2;
-    ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)calloc(1, sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t));
+    ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc(
+        sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
     if (bufPool==NULL) return NULL;
     bufPool->totalBuffers = maxNbBuffers;
     bufPool->nbBuffers = 0;
+    bufPool->cMem = cMem;
     return bufPool;
 }
 
@@ -105,8 +113,8 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
     unsigned u;
     if (!bufPool) return;   /* compatibility with free on NULL */
     for (u=0; utotalBuffers; u++)
-        free(bufPool->bTable[u].start);
-    free(bufPool);
+        ZSTD_free(bufPool->bTable[u].start, bufPool->cMem);
+    ZSTD_free(bufPool, bufPool->cMem);
 }
 
 /* assumption : invocation from main thread only ! */
@@ -115,13 +123,15 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
     if (pool->nbBuffers) {   /* try to use an existing buffer */
         buffer_t const buf = pool->bTable[--(pool->nbBuffers)];
         size_t const availBufferSize = buf.size;
-        if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize))   /* large enough, but not too much */
+        if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize))
+            /* large enough, but not too much */
             return buf;
-        free(buf.start);   /* size conditions not respected : scratch this buffer and create a new one */
+        /* size conditions not respected : scratch this buffer, create new one */
+        ZSTD_free(buf.start, pool->cMem);
     }
     /* create new buffer */
     {   buffer_t buffer;
-        void* const start = malloc(bSize);
+        void* const start = ZSTD_malloc(bSize, pool->cMem);
         if (start==NULL) bSize = 0;
         buffer.start = start;   /* note : start can be NULL if malloc fails ! */
         buffer.size = bSize;
@@ -138,7 +148,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
         return;
     }
     /* Reached bufferPool capacity (should not happen) */
-    free(buf.start);
+    ZSTD_free(buf.start, pool->cMem);
 }
 
 
@@ -147,6 +157,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* pool, buffer_t buf)
 typedef struct {
     unsigned totalCCtx;
     unsigned availCCtx;
+    ZSTD_customMem cMem;
     ZSTD_CCtx* cctx[1];   /* variable size */
 } ZSTDMT_CCtxPool;
 
@@ -158,18 +169,21 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
     unsigned u;
     for (u=0; utotalCCtx; u++)
         ZSTD_freeCCtx(pool->cctx[u]);  /* note : compatible with free on NULL */
-    free(pool);
+    ZSTD_free(pool, pool->cMem);
 }
 
 /* ZSTDMT_createCCtxPool() :
  * implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */
-static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads)
+static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
+                                            ZSTD_customMem cMem)
 {
-    ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) calloc(1, sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*));
+    ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
+        sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem);
     if (!cctxPool) return NULL;
+    cctxPool->cMem = cMem;
     cctxPool->totalCCtx = nbThreads;
     cctxPool->availCCtx = 1;   /* at least one cctx for single-thread mode */
-    cctxPool->cctx[0] = ZSTD_createCCtx();
+    cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem);
     if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
     DEBUGLOG(1, "cctxPool created, with %u threads", nbThreads);
     return cctxPool;
@@ -232,7 +246,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
                  job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
     if (job->cdict) {  /* should only happen for first segment */
         size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize);
-        if (job->cdict) DEBUGLOG(3, "using CDict ");
+        if (job->cdict) DEBUGLOG(3, "using CDict");
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
     } else {  /* srcStart points at reloaded section */
         if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0;  /* ensure no srcSize control */
@@ -293,43 +307,56 @@ struct ZSTDMT_CCtx_s {
     unsigned overlapRLog;
     unsigned long long frameContentSize;
     size_t sectionSize;
+    ZSTD_customMem cMem;
     ZSTD_CDict* cdict;
     ZSTD_CStream* cstream;
 };
 
-ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads)
+ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
 {
-    ZSTDMT_CCtx* cctx;
+    ZSTDMT_CCtx* mtctx;
     U32 const minNbJobs = nbThreads + 2;
     U32 const nbJobsLog2 = ZSTD_highbit32(minNbJobs) + 1;
     U32 const nbJobs = 1 << nbJobsLog2;
-    DEBUGLOG(5, "nbThreads : %u  ; minNbJobs : %u ;  nbJobsLog2 : %u ;  nbJobs : %u  \n",
-            nbThreads, minNbJobs, nbJobsLog2, nbJobs);
+    DEBUGLOG(5, "nbThreads: %u ; minNbJobs: %u ; nbJobsLog2: %u ; nbJobs: %u",
+                nbThreads, minNbJobs, nbJobsLog2, nbJobs);
+
     if ((nbThreads < 1) | (nbThreads > ZSTDMT_NBTHREADS_MAX)) return NULL;
-    cctx = (ZSTDMT_CCtx*) calloc(1, sizeof(ZSTDMT_CCtx));
-    if (!cctx) return NULL;
-    cctx->nbThreads = nbThreads;
-    cctx->jobIDMask = nbJobs - 1;
-    cctx->allJobsCompleted = 1;
-    cctx->sectionSize = 0;
-    cctx->overlapRLog = 3;
-    cctx->factory = POOL_create(nbThreads, 1);
-    cctx->jobs = (ZSTDMT_jobDescription*) malloc(nbJobs * sizeof(*cctx->jobs));
-    cctx->buffPool = ZSTDMT_createBufferPool(nbThreads);
-    cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads);
-    if (!cctx->factory | !cctx->jobs | !cctx->buffPool | !cctx->cctxPool) {
-        ZSTDMT_freeCCtx(cctx);
+    if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL))
+        /* invalid custom allocator */
+        return NULL;
+
+    mtctx = (ZSTDMT_CCtx*) ZSTD_calloc(sizeof(ZSTDMT_CCtx), cMem);
+    if (!mtctx) return NULL;
+    mtctx->cMem = cMem;
+    mtctx->nbThreads = nbThreads;
+    mtctx->jobIDMask = nbJobs - 1;
+    mtctx->allJobsCompleted = 1;
+    mtctx->sectionSize = 0;
+    mtctx->overlapRLog = 3;
+    mtctx->factory = POOL_create(nbThreads, 1);
+    mtctx->jobs = (ZSTDMT_jobDescription*)ZSTD_calloc(
+        nbJobs * sizeof(*mtctx->jobs), cMem);
+    mtctx->buffPool = ZSTDMT_createBufferPool(nbThreads, cMem);
+    mtctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads, cMem);
+    if (!mtctx->factory | !mtctx->jobs | !mtctx->buffPool | !mtctx->cctxPool) {
+        ZSTDMT_freeCCtx(mtctx);
         return NULL;
     }
     if (nbThreads==1) {
-        cctx->cstream = ZSTD_createCStream();
-        if (!cctx->cstream) {
-            ZSTDMT_freeCCtx(cctx); return NULL;
+        mtctx->cstream = ZSTD_createCStream_advanced(cMem);
+        if (!mtctx->cstream) {
+            ZSTDMT_freeCCtx(mtctx); return NULL;
     }   }
-    pthread_mutex_init(&cctx->jobCompleted_mutex, NULL);   /* Todo : check init function return */
-    pthread_cond_init(&cctx->jobCompleted_cond, NULL);
-    DEBUGLOG(4, "mt_cctx created, for %u threads \n", nbThreads);
-    return cctx;
+    pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL);   /* Todo : check init function return */
+    pthread_cond_init(&mtctx->jobCompleted_cond, NULL);
+    DEBUGLOG(4, "mt_cctx created, for %u threads", nbThreads);
+    return mtctx;
+}
+
+ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads)
+{
+    return ZSTDMT_createCCtx_advanced(nbThreads, ZSTD_defaultCMem);
 }
 
 /* ZSTDMT_releaseAllJobResources() :
@@ -357,13 +384,13 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
     POOL_free(mtctx->factory);
     if (!mtctx->allJobsCompleted) ZSTDMT_releaseAllJobResources(mtctx); /* stop workers first */
     ZSTDMT_freeBufferPool(mtctx->buffPool);  /* release job resources into pools first */
-    free(mtctx->jobs);
+    ZSTD_free(mtctx->jobs, mtctx->cMem);
     ZSTDMT_freeCCtxPool(mtctx->cctxPool);
     ZSTD_freeCDict(mtctx->cdict);
     ZSTD_freeCStream(mtctx->cstream);
     pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
     pthread_cond_destroy(&mtctx->jobCompleted_cond);
-    free(mtctx);
+    ZSTD_free(mtctx, mtctx->cMem);
     return 0;
 }
 
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 27f78ee03..92f7d8d0b 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -28,6 +28,7 @@
 
 typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
 ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
+ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem);
 ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx);
 
 ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
diff --git a/lib/zstd.h b/lib/zstd.h
index d3a4d1783..d4c24e7b3 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -421,6 +421,8 @@ typedef struct {
 typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
+/* use this constant to defer to stdlib's functions */
+static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL};
 
 /***************************************
 *  Frame size functions

From 5bcef1ada223c1b1de5634eab4c4c071d9536c5c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 30 May 2017 16:37:19 -0700
Subject: [PATCH 035/109] removed mtctx->cstream

use the first cctx in pool when ZSTDMT is used in single-thread mode
now that cctx and cstream are the same object.
---
 lib/compress/zstdmt_compress.c | 66 ++++++++++++++++++++++------------
 1 file changed, 44 insertions(+), 22 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index bd3de4586..ca5cd6aee 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -27,7 +27,13 @@
 
 
 /* ======   Debug   ====== */
-#if defined(ZSTDMT_DEBUG)
+#if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=1)
+#  include 
+#else
+#  define assert(condition) ((void)0)
+#endif
+
+#if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=2)
 
 #  include 
 #  include 
@@ -309,7 +315,6 @@ struct ZSTDMT_CCtx_s {
     size_t sectionSize;
     ZSTD_customMem cMem;
     ZSTD_CDict* cdict;
-    ZSTD_CStream* cstream;
 };
 
 ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
@@ -343,11 +348,6 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
         ZSTDMT_freeCCtx(mtctx);
         return NULL;
     }
-    if (nbThreads==1) {
-        mtctx->cstream = ZSTD_createCStream_advanced(cMem);
-        if (!mtctx->cstream) {
-            ZSTDMT_freeCCtx(mtctx); return NULL;
-    }   }
     pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL);   /* Todo : check init function return */
     pthread_cond_init(&mtctx->jobCompleted_cond, NULL);
     DEBUGLOG(4, "mt_cctx created, for %u threads", nbThreads);
@@ -387,7 +387,6 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
     ZSTD_free(mtctx->jobs, mtctx->cMem);
     ZSTDMT_freeCCtxPool(mtctx->cctxPool);
     ZSTD_freeCDict(mtctx->cdict);
-    ZSTD_freeCStream(mtctx->cstream);
     pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
     pthread_cond_destroy(&mtctx->jobCompleted_cond);
     ZSTD_free(mtctx, mtctx->cMem);
@@ -540,8 +539,12 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
                                     ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     ZSTD_customMem const cmem = { NULL, NULL, NULL };
-    DEBUGLOG(3, "Started new compression, with windowLog : %u", params.cParams.windowLog);
-    if (zcs->nbThreads==1) return ZSTD_initCStream_advanced(zcs->cstream, dict, dictSize, params, pledgedSrcSize);
+    DEBUGLOG(3, "Started new compression, with windowLog : %u",
+                    params.cParams.windowLog);
+    if (zcs->nbThreads==1)
+        return ZSTD_initCStream_advanced(zcs->cctxPool->cctx[0],
+                                        dict, dictSize,
+                                        params, pledgedSrcSize);
     if (zcs->allJobsCompleted == 0) {   /* previous job not correctly finished */
         ZSTDMT_waitForAllJobsCompleted(zcs);
         ZSTDMT_releaseAllJobResources(zcs);
@@ -587,7 +590,8 @@ size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* zcs,
  * pledgedSrcSize is optional and can be zero == unknown */
 size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
 {
-    if (zcs->nbThreads==1) return ZSTD_resetCStream(zcs->cstream, pledgedSrcSize);
+    if (zcs->nbThreads==1)
+        return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize);
     return ZSTDMT_initCStream_internal(zcs, NULL, 0, 0, zcs->params, pledgedSrcSize);
 }
 
@@ -612,13 +616,16 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
         return ERROR(memory_allocation);
     }
 
-    DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ", zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
+    DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ",
+                zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize);
     zcs->jobs[jobID].src = zcs->inBuff.buffer;
     zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start;
     zcs->jobs[jobID].srcSize = srcSize;
-    zcs->jobs[jobID].dictSize = zcs->dictSize;   /* note : zcs->inBuff.filled is presumed >= srcSize + dictSize */
+    zcs->jobs[jobID].dictSize = zcs->dictSize;
+    assert(zcs->inBuff.filled >= srcSize + zcs->dictSize);
     zcs->jobs[jobID].params = zcs->params;
-    if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0;  /* do not calculate checksum within sections, just keep it in header for first section */
+    /* do not calculate checksum within sections, but write it in header for first section */
+    if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0;
     zcs->jobs[jobID].cdict = zcs->nextJobID==0 ? zcs->cdict : NULL;
     zcs->jobs[jobID].fullFrameSize = zcs->frameContentSize;
     zcs->jobs[jobID].dstBuff = dstBuffer;
@@ -643,8 +650,12 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
         }
         DEBUGLOG(5, "inBuff filled to %u", (U32)zcs->inBuff.filled);
         zcs->inBuff.filled -= srcSize + zcs->dictSize - newDictSize;
-        DEBUGLOG(5, "new job : filled to %u, with %u dict and %u src", (U32)zcs->inBuff.filled, (U32)newDictSize, (U32)(zcs->inBuff.filled - newDictSize));
-        memmove(zcs->inBuff.buffer.start, (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize, zcs->inBuff.filled);
+        DEBUGLOG(5, "new job : filled to %u, with %u dict and %u src",
+                    (U32)zcs->inBuff.filled, (U32)newDictSize,
+                    (U32)(zcs->inBuff.filled - newDictSize));
+        memmove(zcs->inBuff.buffer.start,
+            (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize,
+            zcs->inBuff.filled);
         DEBUGLOG(5, "new inBuff pre-filled");
         zcs->dictSize = newDictSize;
     } else {
@@ -653,10 +664,16 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
         zcs->dictSize = 0;
         zcs->frameEnded = 1;
         if (zcs->nextJobID == 0)
-            zcs->params.fParams.checksumFlag = 0;   /* single chunk : checksum is calculated directly within worker thread */
+            /* single chunk exception : checksum is calculated directly within worker thread */
+            zcs->params.fParams.checksumFlag = 0;
     }
 
-    DEBUGLOG(3, "posting job %u : %u bytes  (end:%u) (note : doneJob = %u=>%u)", zcs->nextJobID, (U32)zcs->jobs[jobID].srcSize, zcs->jobs[jobID].lastChunk, zcs->doneJobID, zcs->doneJobID & zcs->jobIDMask);
+    DEBUGLOG(3, "posting job %u : %u bytes  (end:%u) (note : doneJob = %u=>%u)",
+                zcs->nextJobID,
+                (U32)zcs->jobs[jobID].srcSize,
+                zcs->jobs[jobID].lastChunk,
+                zcs->doneJobID,
+                zcs->doneJobID & zcs->jobIDMask);
     POOL_add(zcs->factory, ZSTDMT_compressChunk, &zcs->jobs[jobID]);   /* this call is blocking when thread worker pool is exhausted */
     zcs->nextJobID++;
     return 0;
@@ -729,8 +746,11 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
 size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
 {
     size_t const newJobThreshold = zcs->dictSize + zcs->targetSectionSize + zcs->marginSize;
-    if (zcs->frameEnded) return ERROR(stage_wrong);   /* current frame being ended. Only flush is allowed. Restart with init */
-    if (zcs->nbThreads==1) return ZSTD_compressStream(zcs->cstream, output, input);
+    if (zcs->frameEnded)
+        /* current frame being ended. Only flush is allowed. Restart with init */
+        return ERROR(stage_wrong);
+    if (zcs->nbThreads==1)
+        return ZSTD_compressStream(zcs->cctxPool->cctx[0], output, input);
 
     /* fill input buffer */
     {   size_t const toLoad = MIN(input->size - input->pos, zcs->inBuffSize - zcs->inBuff.filled);
@@ -770,12 +790,14 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* outp
 
 size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
 {
-    if (zcs->nbThreads==1) return ZSTD_flushStream(zcs->cstream, output);
+    if (zcs->nbThreads==1)
+        return ZSTD_flushStream(zcs->cctxPool->cctx[0], output);
     return ZSTDMT_flushStream_internal(zcs, output, 0);
 }
 
 size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
 {
-    if (zcs->nbThreads==1) return ZSTD_endStream(zcs->cstream, output);
+    if (zcs->nbThreads==1)
+        return ZSTD_endStream(zcs->cctxPool->cctx[0], output);
     return ZSTDMT_flushStream_internal(zcs, output, 1);
 }

From ae728a43b848633cf327f901ea8838f2aa1a1e3d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 30 May 2017 17:11:39 -0700
Subject: [PATCH 036/109] removed defaultCustomMem

now ZSTD_customCMem is promoted as new default.

Advantages : ZSTD_customCMem = { NULL, NULL, NULL},
so it's natural default after a memset.

ZSTD_customCMem is public constant
(defaultCustomMem was private only).

Also : makes it possible to introduce ZSTD_calloc(),
which can now default to stdlib's calloc()
when it detects system default.

Fixed zlibwrapper which depended on defaultCustomMem.
---
 lib/common/zstd_common.c         | 14 --------------
 lib/common/zstd_internal.h       |  5 -----
 lib/compress/zstd_compress.c     | 17 +++++++----------
 lib/compress/zstdmt_compress.h   |  1 +
 lib/decompress/zstd_decompress.c | 11 ++++-------
 zlibWrapper/zstd_zlibwrapper.c   | 11 ++++-------
 6 files changed, 16 insertions(+), 43 deletions(-)

diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 9b46df1d8..801483960 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -50,20 +50,6 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString
 /*=**************************************************************
 *  Custom allocator
 ****************************************************************/
-/* default uses stdlib */
-void* ZSTD_defaultAllocFunction(void* opaque, size_t size)
-{
-    void* address = malloc(size);
-    (void)opaque;
-    return address;
-}
-
-void ZSTD_defaultFreeFunction(void* opaque, void* address)
-{
-    (void)opaque;
-    free(address);
-}
-
 void* ZSTD_malloc(size_t size, ZSTD_customMem customMem)
 {
     if (customMem.customAlloc)
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 5ab4aaeff..fedaac697 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -238,11 +238,6 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);
 int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
 
 /* custom memory allocation functions */
-void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
-void ZSTD_defaultFreeFunction(void* opaque, void* address);
-#ifndef ZSTD_DLL_IMPORT
-static const ZSTD_customMem defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
-#endif
 void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
 void* ZSTD_calloc(size_t size, ZSTD_customMem customMem);
 void ZSTD_free(void* ptr, ZSTD_customMem customMem);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1feec765e..7c2fd520f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -158,15 +158,14 @@ struct ZSTD_CCtx_s {
 
 ZSTD_CCtx* ZSTD_createCCtx(void)
 {
-    return ZSTD_createCCtx_advanced(defaultCustomMem);
+    return ZSTD_createCCtx_advanced(ZSTD_defaultCMem);
 }
 
 ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
 {
     ZSTD_CCtx* cctx;
 
-    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-    if (!customMem.customAlloc || !customMem.customFree) return NULL;
+    if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
 
     cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
     if (!cctx) return NULL;
@@ -3225,9 +3224,9 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
     size_t result;
     ZSTD_CCtx ctxBody;
     memset(&ctxBody, 0, sizeof(ctxBody));
-    memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
+    ctxBody.customMem = ZSTD_defaultCMem;
     result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
-    ZSTD_free(ctxBody.workSpace, defaultCustomMem);  /* can't free ctxBody itself, as it's on stack; free only heap content */
+    ZSTD_free(ctxBody.workSpace, ZSTD_defaultCMem);  /* can't free ctxBody itself, as it's on stack; free only heap content */
     return result;
 }
 
@@ -3288,8 +3287,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
                                       ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
 {
     DEBUGLOG(5, "ZSTD_createCDict_advanced");
-    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-    if (!customMem.customAlloc || !customMem.customFree) return NULL;
+    if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
 
     {   ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
         ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
@@ -3442,12 +3440,11 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
 
 ZSTD_CStream* ZSTD_createCStream(void)
 {
-    return ZSTD_createCStream_advanced(defaultCustomMem);
+    return ZSTD_createCStream_advanced(ZSTD_defaultCMem);
 }
 
 ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
-{
-    /* CStream and CCtx are now same object */
+{   /* CStream and CCtx are now same object */
     return ZSTD_createCCtx_advanced(customMem);
 }
 
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 92f7d8d0b..94fef7c07 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -57,6 +57,7 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
 ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize,  /**< dict can be released after init, a local copy is preserved within zcs */
                                           ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
 
+
 /* ZSDTMT_parameter :
  * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
 typedef enum {
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index a59724386..7dfcddb1b 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -206,9 +206,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
 {
     ZSTD_DCtx* dctx;
 
-    if (!customMem.customAlloc && !customMem.customFree)
-        customMem = defaultCustomMem;
-    if (!customMem.customAlloc || !customMem.customFree)
+    if (!customMem.customAlloc ^ !customMem.customFree)
         return NULL;
 
     dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
@@ -233,7 +231,7 @@ ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
 
 ZSTD_DCtx* ZSTD_createDCtx(void)
 {
-    return ZSTD_createDCtx_advanced(defaultCustomMem);
+    return ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
 }
 
 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
@@ -2012,8 +2010,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_
 
 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
 {
-    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-    if (!customMem.customAlloc || !customMem.customFree) return NULL;
+    if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
 
     {   ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
         if (!ddict) return NULL;
@@ -2155,7 +2152,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
 
 ZSTD_DStream* ZSTD_createDStream(void)
 {
-    return ZSTD_createDStream_advanced(defaultCustomMem);
+    return ZSTD_createDStream_advanced(ZSTD_defaultCMem);
 }
 
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 1960d19e9..a078cf556 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -8,6 +8,7 @@
  */
 
 
+#include 
 #include                  /* vsprintf */
 #include                 /* va_list, for z_gzprintf */
 #define NO_DUMMY_DECL
@@ -117,10 +118,8 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
           memcpy(&zwc->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
         }
     } else {
-        zwc = (ZWRAP_CCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_CCtx));
+        zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc));
         if (zwc==NULL) return NULL;
-        memset(zwc, 0, sizeof(ZWRAP_CCtx));
-        memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
     }
 
     return zwc;
@@ -219,7 +218,7 @@ int ZWRAP_deflateReset_keepDict(z_streamp strm)
         return deflateReset(strm);
 
     { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
-      if (zwc) { 
+      if (zwc) {
           zwc->streamEnd = 0;
           zwc->totalInBytes = 0;
       }
@@ -459,10 +458,8 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
           memcpy(&zwd->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
         }
     } else {
-        zwd = (ZWRAP_DCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_DCtx));
+        zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd));
         if (zwd==NULL) return NULL;
-        memset(zwd, 0, sizeof(ZWRAP_DCtx));
-        memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
     }
 
     MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN);   /* if compilation fails here, assertion is false */

From deee6e523f81671e84f93ceb6be3e1a47387cc25 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 30 May 2017 17:42:00 -0700
Subject: [PATCH 037/109] expose ZSTD_compress_generic_simpleArgs()

which is a binding towards ZSTD_compress_generic()
using only integral types for arguments.
---
 doc/zstd_manual.html           | 13 +++++++++
 lib/compress/zstd_compress.c   | 51 +++++++++++++++++++---------------
 lib/compress/zstdmt_compress.c |  3 +-
 lib/zstd.h                     | 14 ++++++++++
 4 files changed, 57 insertions(+), 24 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 2c66409cb..3b7f32870 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -698,6 +698,19 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
  
 


 
+
size_t ZSTD_compress_generic_simpleArgs (
+                ZSTD_CCtx* cctx,
+                void* dst, size_t dstCapacity, size_t* dstPos,
+          const void* src, size_t srcSize, size_t* srcPos,
+                ZSTD_EndDirective endOp);
+
  Same as ZSTD_compress_generic(),
+  but using only simple integral types as arguments.
+  Argument list is less expressive than ZSTD_{in,out}Buffer,
+  but can be helpful for binders towards dynamic languages
+  which have troubles handling structures containing memory pointers.
+ 
+


+
 
Advanced decompression functions

 
 
unsigned ZSTD_isFrame(const void* buffer, size_t size);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 7c2fd520f..b046a2c5f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3710,49 +3710,54 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf
     return result;
 }
 
-size_t ZSTD_compress_generic_integral (
-                            ZSTD_CCtx* cctx,
-                            void* dst, size_t dstCapacity, size_t* dstPos,
-                      const void* src, size_t srcSize, size_t* srcPos,
-                            ZSTD_EndDirective endOp)
+size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
+                              ZSTD_outBuffer* output,
+                              ZSTD_inBuffer* input,
+                              ZSTD_EndDirective endOp)
 {
     /* check conditions */
-    if (*dstPos > dstCapacity) return ERROR(GENERIC);
-    if (*srcPos > srcSize) return ERROR(GENERIC);
+    if (output->pos > output->size) return ERROR(GENERIC);
+    if (input->pos  > input->size)  return ERROR(GENERIC);
 
     assert(cctx!=NULL);
     if (cctx->streamStage == zcss_init) {
         /* transparent reset */
         ZSTD_parameters params = cctx->requestedParams;
-        DEBUGLOG(5, "ZSTD_compress_generic_integral : transparent reset");
+        DEBUGLOG(5, "ZSTD_compress_generic : transparent reset");
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
                                     cctx->frameContentSize, 0 /* dictSize */);
         CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->frameContentSize) );
     }
 
-    {   size_t sizeRead = srcSize - *srcPos;
-        size_t sizeWritten = dstCapacity - *dstPos;
+    {   size_t sizeRead = input->size - input->pos;
+        size_t sizeWritten = output->size - output->pos;
         DEBUGLOG(5, "starting ZSTD_compressStream_generic");
         CHECK_F( ZSTD_compressStream_generic(cctx,
-                        (char*)dst + *dstPos, &sizeWritten,
-                        (const char*)src + *srcPos, &sizeRead, endOp) );
-        *srcPos += sizeRead;
-        *dstPos += sizeWritten;
+                        (char*)output->dst + output->pos, &sizeWritten,
+                        (const char*)input->src + input->pos, &sizeRead, endOp) );
+        input->pos += sizeRead;
+        output->pos += sizeWritten;
     }
     DEBUGLOG(5, "completing ZSTD_compress_generic_integral");
-    return cctx->outBuffContentSize - cctx->outBuffFlushedSize;   /* remaining to flush */
+    return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
 }
 
-size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
-                              ZSTD_outBuffer* output,
-                              ZSTD_inBuffer* input,
-                              ZSTD_EndDirective endOp)
+size_t ZSTD_compress_generic_simpleArgs (
+                            ZSTD_CCtx* cctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos,
+                            ZSTD_EndDirective endOp)
 {
-    return ZSTD_compress_generic_integral(cctx,
-                output->dst, output->size, &output->pos,
-                input->src, input->size, &input->pos,
-                endOp);
+    ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
+    ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
+
+    size_t const hint = ZSTD_compress_generic(cctx, &output, &input, endOp);
+    if (ZSTD_isError(hint)) return hint;
+
+    *dstPos = output.pos;
+    *srcPos = input.pos;
+    return hint;
 }
 
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index ca5cd6aee..b2c82e489 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -783,7 +783,8 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* outp
     }
 
     /* check if there is any data available to flush */
-    DEBUGLOG(5, "zcs->doneJobID : %u  ; zcs->nextJobID : %u ", zcs->doneJobID, zcs->nextJobID);
+    DEBUGLOG(5, "zcs->doneJobID : %u  ; zcs->nextJobID : %u",
+            zcs->doneJobID, zcs->nextJobID);
     return ZSTDMT_flushNextJob(zcs, output, 1);
 }
 
diff --git a/lib/zstd.h b/lib/zstd.h
index d4c24e7b3..1fc35fc55 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -823,6 +823,20 @@ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
 ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
 
 
+/*! ZSTD_compress_generic_simpleArgs() :
+ *  Same as ZSTD_compress_generic(),
+ *  but using only simple integral types as arguments.
+ *  Argument list is less expressive than ZSTD_{in,out}Buffer,
+ *  but can be helpful for binders towards dynamic languages
+ *  which have troubles handling structures containing memory pointers.
+ */
+size_t ZSTD_compress_generic_simpleArgs (
+                            ZSTD_CCtx* cctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos,
+                            ZSTD_EndDirective endOp);
+
+
 
 /*--- Advanced decompression functions ---*/
 

From c4f46b94ce6ba497c75b0b44d9e45a0d2872602d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 30 May 2017 17:45:37 -0700
Subject: [PATCH 038/109] ZSTD_createCCtx_advanced() now uses ZSTD_calloc()

initially uses calloc() instead of memset().

Performance improvement is unlikely measurable,
since ZSTD_CCtx is now very small,
with all tables transferred into workSpace.
---
 lib/compress/zstd_compress.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b046a2c5f..00a93e6e5 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -167,9 +167,8 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
 
     if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
 
-    cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
+    cctx = (ZSTD_CCtx*) ZSTD_calloc(sizeof(ZSTD_CCtx), customMem);
     if (!cctx) return NULL;
-    memset(cctx, 0, sizeof(ZSTD_CCtx));
     cctx->customMem = customMem;
     cctx->compressionLevel = ZSTD_CLEVEL_DEFAULT;
     return cctx;

From 01b1549f83e58db9494234285fbf6673b2ed7f04 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 30 May 2017 18:10:26 -0700
Subject: [PATCH 039/109] finally converted ZSTD_compressStream_generic() to
 use {in,ou}Buffer

replacing the older read/write variables from ZBUFF_* era.
Mostly to help code readability.

Fixed relevant callers.
---
 lib/compress/zstd_compress.c | 96 +++++++++++++++---------------------
 1 file changed, 41 insertions(+), 55 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 00a93e6e5..90ee1bb6d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3571,7 +3571,8 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 
 /*======   Compression   ======*/
 
-MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity,
+                           const void* src, size_t srcSize)
 {
     size_t const length = MIN(dstCapacity, srcSize);
     memcpy(dst, src, length);
@@ -3579,18 +3580,19 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
 }
 
 static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
-                              void* dst, size_t* dstCapacityPtr,
-                        const void* src, size_t* srcSizePtr,
-                              ZSTD_EndDirective const flushMode)
+                                        ZSTD_outBuffer* output,
+                                        ZSTD_inBuffer* input,
+                                        ZSTD_EndDirective const flushMode)
 {
     U32 someMoreWork = 1;
-    const char* const istart = (const char*)src;
-    const char* const iend = istart + *srcSizePtr;
-    const char* ip = istart;
-    char* const ostart = (char*)dst;
-    char* const oend = ostart + *dstCapacityPtr;
-    char* op = ostart;
+    const char* const istart = (const char*)input->src;
+    const char* const iend = istart + input->size;
+    const char* ip = istart + input->pos;
+    char* const ostart = (char*)output->dst;
+    char* const oend = ostart + output->size;
+    char* op = ostart + output->pos;
 
+    /* expected to be already allocated */
     assert(zcs->inBuff != NULL);
     assert(zcs->outBuff!= NULL);
 
@@ -3604,7 +3606,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
         case zcss_load:
             /* complete inBuffer */
             {   size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
-                size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip);
+                size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos,
+                                                    toLoad, ip, iend-ip);
                 zcs->inBuffPos += loaded;
                 ip += loaded;
                 if ( (flushMode == ZSTD_e_continue)
@@ -3663,11 +3666,16 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
         case zcss_flush:
             DEBUGLOG(5, "flush stage");
             {   size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
-                size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
+                size_t const flushed = ZSTD_limitCopy(op, oend-op,
+                            zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
                 DEBUGLOG(5, "toFlush: %u  ; flushed: %u", (U32)toFlush, (U32)flushed);
                 op += flushed;
                 zcs->outBuffFlushedSize += flushed;
-                if (toFlush!=flushed) { someMoreWork = 0; break; }  /* dst too small to store flushed data : stop there */
+                if (toFlush!=flushed) {
+                    /* dst too small to store flushed data : stop there */
+                    someMoreWork = 0;
+                    break;
+                }
                 zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
                 if (zcs->frameEnded) {
                     DEBUGLOG(5, "Frame completed");
@@ -3687,8 +3695,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
         }
     }
 
-    *srcSizePtr = ip - istart;
-    *dstCapacityPtr = op - ostart;
+    input->pos = ip - istart;
+    output->pos = op - ostart;
     if (zcs->frameEnded) return 0;
     {   size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
         if (hintInSize==0) hintInSize = zcs->blockSize;
@@ -3698,15 +3706,11 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
 
 size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
 {
-    size_t sizeRead = input->size - input->pos;
-    size_t sizeWritten = output->size - output->pos;
-    size_t const result = ZSTD_compressStream_generic(
-                            zcs,
-                            (char*)(output->dst) + output->pos, &sizeWritten,
-                            (const char*)(input->src) + input->pos, &sizeRead, ZSTD_e_continue);
-    input->pos += sizeRead;
-    output->pos += sizeWritten;
-    return result;
+    /* check conditions */
+    if (output->pos > output->size) return ERROR(GENERIC);
+    if (input->pos  > input->size)  return ERROR(GENERIC);
+
+    return ZSTD_compressStream_generic(zcs, output, input, ZSTD_e_continue);
 }
 
 size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
@@ -3717,8 +3721,8 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     /* check conditions */
     if (output->pos > output->size) return ERROR(GENERIC);
     if (input->pos  > input->size)  return ERROR(GENERIC);
-
     assert(cctx!=NULL);
+
     if (cctx->streamStage == zcss_init) {
         /* transparent reset */
         ZSTD_parameters params = cctx->requestedParams;
@@ -3729,15 +3733,9 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->frameContentSize) );
     }
 
-    {   size_t sizeRead = input->size - input->pos;
-        size_t sizeWritten = output->size - output->pos;
-        DEBUGLOG(5, "starting ZSTD_compressStream_generic");
-        CHECK_F( ZSTD_compressStream_generic(cctx,
-                        (char*)output->dst + output->pos, &sizeWritten,
-                        (const char*)input->src + input->pos, &sizeRead, endOp) );
-        input->pos += sizeRead;
-        output->pos += sizeWritten;
-    }
+    DEBUGLOG(5, "starting ZSTD_compressStream_generic");
+    CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
+
     DEBUGLOG(5, "completing ZSTD_compress_generic_integral");
     return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
 }
@@ -3750,13 +3748,14 @@ size_t ZSTD_compress_generic_simpleArgs (
 {
     ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
     ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
-
+    /* ZSTD_compress_generic() will check validity of dstPos and srcPos */
     size_t const hint = ZSTD_compress_generic(cctx, &output, &input, endOp);
     if (ZSTD_isError(hint)) return hint;
 
     *dstPos = output.pos;
     *srcPos = input.pos;
     return hint;
+
 }
 
 
@@ -3766,34 +3765,21 @@ size_t ZSTD_compress_generic_simpleArgs (
 *   @return : amount of data remaining to flush */
 size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
-    size_t srcSize = 0;
-    size_t sizeWritten = output->size - output->pos;
-    size_t const result = ZSTD_compressStream_generic(zcs,
-                                (char*)(output->dst) + output->pos, &sizeWritten,
-                                &srcSize, &srcSize, /* use a valid src address instead of NULL */
-                                ZSTD_e_flush);
-    output->pos += sizeWritten;
-    if (ZSTD_isError(result)) return result;
-    return zcs->outBuffContentSize - zcs->outBuffFlushedSize;   /* remaining to flush */
+    ZSTD_inBuffer input = { &input, 0, 0 };
+    if (output->pos > output->size) return ERROR(GENERIC);
+    CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_flush) );
+    return zcs->outBuffContentSize - zcs->outBuffFlushedSize;  /* remaining to flush */
 }
 
 
 size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
-    BYTE* const ostart = (BYTE*)(output->dst) + output->pos;
-    size_t srcSize = 0;
-    size_t sizeWritten = output->size - output->pos;
-
-    size_t const result = ZSTD_compressStream_generic(zcs,
-                                ostart, &sizeWritten,
-                                &srcSize /* valid address */, &srcSize,
-                                ZSTD_e_end);
-    output->pos += sizeWritten;
-    if (ZSTD_isError(result)) return result;
+    ZSTD_inBuffer input = { &input, 0, 0 };
+    if (output->pos > output->size) return ERROR(GENERIC);
+    CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_end) );
 
     DEBUGLOG(5, "ZSTD_endStream : remaining to flush : %u",
             (unsigned)(zcs->outBuffContentSize - zcs->outBuffFlushedSize));
-
     return zcs->outBuffContentSize - zcs->outBuffFlushedSize;
 }
 

From 9a691e0f55062156c2f5902405fe0d9b9394a9ab Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 31 May 2017 01:17:44 -0700
Subject: [PATCH 040/109] fixed visual warnings

---
 lib/common/zstd_internal.h   | 1 +
 lib/compress/zstd_compress.c | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index fedaac697..6f9b4ef1f 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -18,6 +18,7 @@
 #  include                     /* For Visual 2005 */
 #  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
 #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
 #else
 #  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 90ee1bb6d..d57d909a7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3765,7 +3765,8 @@ size_t ZSTD_compress_generic_simpleArgs (
 *   @return : amount of data remaining to flush */
 size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
-    ZSTD_inBuffer input = { &input, 0, 0 };
+    const char dummy[8] = {0};
+    ZSTD_inBuffer input = { dummy, 0, 0 };
     if (output->pos > output->size) return ERROR(GENERIC);
     CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_flush) );
     return zcs->outBuffContentSize - zcs->outBuffFlushedSize;  /* remaining to flush */
@@ -3774,7 +3775,8 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 
 size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
-    ZSTD_inBuffer input = { &input, 0, 0 };
+    const char dummy[8] = {0};
+    ZSTD_inBuffer input = { dummy, 0, 0 };
     if (output->pos > output->size) return ERROR(GENERIC);
     CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_end) );
 

From 18ab5affa509170bbd5226a1bcde732bedcb3d65 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 31 May 2017 09:59:22 -0700
Subject: [PATCH 041/109] fixed visual warning

---
 lib/compress/zstd_compress.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 90ee1bb6d..fa574a226 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3575,7 +3575,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity,
                            const void* src, size_t srcSize)
 {
     size_t const length = MIN(dstCapacity, srcSize);
-    memcpy(dst, src, length);
+    if (length) memcpy(dst, src, length);
     return length;
 }
 
@@ -3765,7 +3765,7 @@ size_t ZSTD_compress_generic_simpleArgs (
 *   @return : amount of data remaining to flush */
 size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
-    ZSTD_inBuffer input = { &input, 0, 0 };
+    ZSTD_inBuffer input = { NULL, 0, 0 };
     if (output->pos > output->size) return ERROR(GENERIC);
     CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_flush) );
     return zcs->outBuffContentSize - zcs->outBuffFlushedSize;  /* remaining to flush */
@@ -3774,7 +3774,7 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 
 size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
-    ZSTD_inBuffer input = { &input, 0, 0 };
+    ZSTD_inBuffer input = { NULL, 0, 0 };
     if (output->pos > output->size) return ERROR(GENERIC);
     CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_end) );
 

From cd2892fd1eddc9e4cb7375b2a0167668c69a1981 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 1 Jun 2017 09:44:54 -0700
Subject: [PATCH 042/109] protected impossible switch(){default:} with
 assert(0)

can be converted into assume(0) in some future
---
 lib/compress/zstd_compress.c | 125 ++++++++++++++++++-----------------
 1 file changed, 65 insertions(+), 60 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ddbeaf56e..175a7b2b0 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -265,80 +265,82 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
     switch(param)
     {
     case ZSTD_p_compressionLevel :
-            if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel();   /* cap max compression level */
-            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
-            cctx->compressionLevel = value;
-            return 0;
+        if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel();   /* cap max compression level */
+        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        cctx->compressionLevel = value;
+        return 0;
 
     case ZSTD_p_windowLog :
-            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
-            CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
-            ZSTD_cLevelToCParams(cctx);
-            cctx->requestedParams.cParams.windowLog = value;
-            return 0;
+        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+        ZSTD_cLevelToCParams(cctx);
+        cctx->requestedParams.cParams.windowLog = value;
+        return 0;
 
     case ZSTD_p_hashLog :
-            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
-            CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
-            ZSTD_cLevelToCParams(cctx);
-            cctx->requestedParams.cParams.hashLog = value;
-            return 0;
+        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
+        ZSTD_cLevelToCParams(cctx);
+        cctx->requestedParams.cParams.hashLog = value;
+        return 0;
 
     case ZSTD_p_chainLog :
-            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
-            CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
-            ZSTD_cLevelToCParams(cctx);
-            cctx->requestedParams.cParams.chainLog = value;
-            return 0;
+        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
+        ZSTD_cLevelToCParams(cctx);
+        cctx->requestedParams.cParams.chainLog = value;
+        return 0;
 
     case ZSTD_p_searchLog :
-            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
-            CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
-            ZSTD_cLevelToCParams(cctx);
-            cctx->requestedParams.cParams.searchLog = value;
-            return 0;
+        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
+        ZSTD_cLevelToCParams(cctx);
+        cctx->requestedParams.cParams.searchLog = value;
+        return 0;
 
     case ZSTD_p_minMatch :
-            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
-            CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
-            ZSTD_cLevelToCParams(cctx);
-            cctx->requestedParams.cParams.searchLength = value;
-            return 0;
+        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+        ZSTD_cLevelToCParams(cctx);
+        cctx->requestedParams.cParams.searchLength = value;
+        return 0;
 
     case ZSTD_p_targetLength :
-            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
-            CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
-            ZSTD_cLevelToCParams(cctx);
-            cctx->requestedParams.cParams.targetLength = value;
-            return 0;
+        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
+        ZSTD_cLevelToCParams(cctx);
+        cctx->requestedParams.cParams.targetLength = value;
+        return 0;
 
     case ZSTD_p_compressionStrategy :
-            if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
-            CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
-            ZSTD_cLevelToCParams(cctx);
-            cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value;
-            return 0;
+        if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
+        ZSTD_cLevelToCParams(cctx);
+        cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value;
+        return 0;
 
 #if 0
     case ZSTD_p_windowSize :   /* to be done later */
-            return ERROR(compressionParameter_unsupported);
+        return ERROR(compressionParameter_unsupported);
 #endif
 
-    case ZSTD_p_contentSizeFlag : /* Content size will be written in frame header _when known_ (default:1) */
-            cctx->requestedParams.fParams.contentSizeFlag = value>0;
-            return 0;
+    case ZSTD_p_contentSizeFlag :
+        /* Content size written in frame header _when known_ (default:1) */
+        cctx->requestedParams.fParams.contentSizeFlag = value>0;
+        return 0;
 
-    case ZSTD_p_checksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
-            cctx->requestedParams.fParams.checksumFlag = value>0;
-            return 0;
+    case ZSTD_p_checksumFlag :
+        /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
+        cctx->requestedParams.fParams.checksumFlag = value>0;
+        return 0;
 
     case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
-            DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
-            cctx->requestedParams.fParams.noDictIDFlag = (value==0);
-            return 0;
+        DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
+        cctx->requestedParams.fParams.noDictIDFlag = (value==0);
+        return 0;
 
     case ZSTD_p_refDictContent :   /* to be done later */
-            return ERROR(compressionParameter_unsupported);
+        return ERROR(compressionParameter_unsupported);
 
     case ZSTD_p_forceMaxWindow :  /* Force back-references to remain < windowSize,
                                    * even when referencing into Dictionary content
@@ -348,8 +350,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
             return 0;
 
     case ZSTD_p_rawContentDict :  /* load dictionary in "content-only" mode (no header analysis) (default:0) */
-            cctx->forceRawDict = value>0;
-            return 0;
+        cctx->forceRawDict = value>0;
+        return 0;
 
     default: return ERROR(parameter_unknown);
     }
@@ -811,10 +813,11 @@ static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void
         case 2: /* 2 - 2 - 12 */
             MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4)));
             break;
-        default:   /*note : should not be necessary : flSize is within {1,2,3} */
         case 3: /* 2 - 2 - 20 */
             MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4)));
             break;
+        default:   /* not necessary : flSize is {1,2,3} */
+            assert(0);
     }
 
     memcpy(ostart + flSize, src, srcSize);
@@ -836,10 +839,11 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons
         case 2: /* 2 - 2 - 12 */
             MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4)));
             break;
-        default:   /*note : should not be necessary : flSize is necessarily within {1,2,3} */
         case 3: /* 2 - 2 - 20 */
             MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4)));
             break;
+        default:   /* not necessary : flSize is {1,2,3} */
+            assert(0);
     }
 
     ostart[flSize] = *(const BYTE*)src;
@@ -901,13 +905,14 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
             MEM_writeLE32(ostart, lhc);
             break;
         }
-    default:   /* should not be necessary, lhSize is only {3,4,5} */
     case 5: /* 2 - 2 - 18 - 18 */
         {   U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
             MEM_writeLE32(ostart, lhc);
             ostart[4] = (BYTE)(cLitSize >> 10);
             break;
         }
+    default:   /* not possible : lhSize is {3,4,5} */
+        assert(0);
     }
     return lhSize+cLitSize;
 }
@@ -2814,7 +2819,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     if (!singleSegment) op[pos++] = windowLogByte;
     switch(dictIDSizeCode)
     {
-        default:   /* impossible */
+        default:  assert(0); /* impossible */
         case 0 : break;
         case 1 : op[pos] = (BYTE)(dictID); pos++; break;
         case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break;
@@ -2822,7 +2827,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
     }
     switch(fcsCode)
     {
-        default:   /* impossible */
+        default:  assert(0); /* impossible */
         case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break;
         case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
         case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break;
@@ -2951,7 +2956,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
         break;
 
     default:
-        return ERROR(GENERIC);   /* strategy doesn't exist; impossible */
+        assert(0);  /* not possible : not a valid strategy id */
     }
 
     zc->nextToUpdate = (U32)(iend - zc->base);
@@ -3691,8 +3696,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
         case zcss_final:
             someMoreWork = 0; break;  /* useless */
 
-        default:
-            return ERROR(GENERIC);   /* impossible */
+        default: /* impossible */
+            assert(0);
         }
     }
 

From c4a5a21c5c4e75b24839267a4f0472948a0159cf Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 1 Jun 2017 17:56:14 -0700
Subject: [PATCH 043/109] created ZSTDMT_sizeof_CCtx() and POOL_sizeof()

required by ZSTD_sizeofCCtx() while adding a ZSTDMT_CCtx*
---
 lib/common/pool.c              |  7 ++++++
 lib/common/pool.h              |  5 +++++
 lib/compress/zstd_compress.c   | 12 ++++++++--
 lib/compress/zstdmt_compress.c | 40 +++++++++++++++++++++++++++++++++-
 lib/compress/zstdmt_compress.h | 17 ++++++++++-----
 5 files changed, 73 insertions(+), 8 deletions(-)

diff --git a/lib/common/pool.c b/lib/common/pool.c
index e439fe1b0..05adcf1ce 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -146,6 +146,13 @@ void POOL_free(POOL_ctx *ctx) {
     free(ctx);
 }
 
+size_t POOL_sizeof(POOL_ctx *ctx) {
+    if (ctx==NULL) return 0;  /* supports sizeof NULL */
+    return sizeof(*ctx)
+        + ctx->queueSize * sizeof(POOL_job)
+        + ctx->numThreads * sizeof(pthread_t);
+}
+
 void POOL_add(void *ctxVoid, POOL_function function, void *opaque) {
     POOL_ctx *ctx = (POOL_ctx *)ctxVoid;
     if (!ctx) { return; }
diff --git a/lib/common/pool.h b/lib/common/pool.h
index 50cb25b12..386cd674b 100644
--- a/lib/common/pool.h
+++ b/lib/common/pool.h
@@ -32,6 +32,11 @@ POOL_ctx *POOL_create(size_t numThreads, size_t queueSize);
 */
 void POOL_free(POOL_ctx *ctx);
 
+/*! POOL_sizeof() :
+    return memory usage of pool returned by POOL_create().
+*/
+size_t POOL_sizeof(POOL_ctx *ctx);
+
 /*! POOL_function :
     The function type that can be added to a thread pool.
 */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 175a7b2b0..7924968e4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -25,6 +25,7 @@
 #define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
 #include "zstd_internal.h"  /* includes zstd.h */
+#include "zstdmt_compress.h"
 
 
 /*-*************************************
@@ -154,8 +155,12 @@ struct ZSTD_CCtx_s {
     size_t outBuffFlushedSize;
     ZSTD_cStreamStage streamStage;
     U32    frameEnded;
+
+    /* Multi-threading */
+    ZSTDMT_CCtx* mtctx;
 };
 
+
 ZSTD_CCtx* ZSTD_createCCtx(void)
 {
     return ZSTD_createCCtx_advanced(ZSTD_defaultCMem);
@@ -205,11 +210,13 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
 size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
 {
     if (cctx==NULL) return 0;   /* support free on NULL */
-    assert(!cctx->staticSize);  /* not compatible with static CCtx */
+    if (cctx->staticSize) return ERROR(memory_allocation);   /* not compatible with static CCtx */
     ZSTD_free(cctx->workSpace, cctx->customMem);
     cctx->workSpace = NULL;
     ZSTD_freeCDict(cctx->cdictLocal);
     cctx->cdictLocal = NULL;
+    ZSTDMT_freeCCtx(cctx->mtctx);
+    cctx->mtctx = NULL;
     ZSTD_free(cctx, cctx->customMem);
     return 0;   /* reserved as a potential error code in the future */
 }
@@ -219,7 +226,8 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
     if (cctx==NULL) return 0;   /* support sizeof on NULL */
     return sizeof(*cctx) + cctx->workSpaceSize
            + ZSTD_sizeof_CDict(cctx->cdictLocal)
-           + cctx->outBuffSize + cctx->inBuffSize;
+           + cctx->outBuffSize + cctx->inBuffSize
+           + ZSTDMT_sizeof_CCtx(cctx->mtctx);
 }
 
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index b2c82e489..57c0360b0 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -123,6 +123,19 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
     ZSTD_free(bufPool, bufPool->cMem);
 }
 
+/* only works at initialization, not during compression */
+static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
+{
+    size_t const poolSize = sizeof(*bufPool)
+                            + (bufPool->totalBuffers - 1) * sizeof(buffer_t);
+    unsigned u;
+    size_t totalBufferSize = 0;
+    for (u=0; utotalBuffers; u++)
+        totalBufferSize += bufPool->bTable[u].size;
+
+    return poolSize + totalBufferSize;
+}
+
 /* assumption : invocation from main thread only ! */
 static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* pool, size_t bSize)
 {
@@ -181,7 +194,7 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
 /* ZSTDMT_createCCtxPool() :
  * implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */
 static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
-                                            ZSTD_customMem cMem)
+                                              ZSTD_customMem cMem)
 {
     ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
         sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem);
@@ -195,6 +208,20 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
     return cctxPool;
 }
 
+/* only works during initialization phase, not during compression */
+static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
+{
+    unsigned const nbThreads = cctxPool->totalCCtx;
+    size_t const poolSize = sizeof(*cctxPool)
+                            + (nbThreads-1)*sizeof(ZSTD_CCtx*);
+    unsigned u;
+    size_t totalCCtxSize = 0;
+    for (u=0; ucctx[u]);
+
+    return poolSize + totalCCtxSize;
+}
+
 static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* pool)
 {
     if (pool->availCCtx) {
@@ -393,6 +420,17 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
     return 0;
 }
 
+size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
+{
+    if (mtctx == NULL) return 0;   /* supports sizeof NULL */
+    return sizeof(*mtctx)
+        + POOL_sizeof(mtctx->factory)
+        + ZSTDMT_sizeof_bufferPool(mtctx->buffPool)
+        + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription)
+        + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool)
+        + ZSTD_sizeof_CDict(mtctx->cdict);
+}
+
 size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value)
 {
     switch(parameter)
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 94fef7c07..ff7f21687 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -24,14 +24,21 @@
 #include "zstd.h"     /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
 
 
-/* ===   Simple one-pass functions   === */
-
+/* ===   Memory management   === */
 typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
 ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads);
-ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem);
-ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* cctx);
+ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads,
+                                                    ZSTD_customMem cMem);
+ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
 
-ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* cctx,
+ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
+ZSTDLIB_API size_t ZSTDMT_estimateCCtxSize(ZSTD_compressionParameters cParams,
+                                           unsigned nbThreads);
+
+
+/* ===   Simple buffer-to-butter one-pass function   === */
+
+ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
                            void* dst, size_t dstCapacity,
                      const void* src, size_t srcSize,
                            int compressionLevel);

From c35e535002bf6b178f6c5ff53ce9b348fa8de427 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 1 Jun 2017 18:44:06 -0700
Subject: [PATCH 044/109] added support for multithreading parameters

---
 lib/compress/zstd_compress.c   | 31 ++++++++++++++++++++++++++++---
 lib/compress/zstdmt_compress.c |  2 +-
 lib/compress/zstdmt_compress.h |  2 +-
 lib/zstd.h                     | 13 +++++++------
 4 files changed, 37 insertions(+), 11 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 7924968e4..8f7c200b2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -157,6 +157,7 @@ struct ZSTD_CCtx_s {
     U32    frameEnded;
 
     /* Multi-threading */
+    U32 nbThreads;
     ZSTDMT_CCtx* mtctx;
 };
 
@@ -353,9 +354,33 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
     case ZSTD_p_forceMaxWindow :  /* Force back-references to remain < windowSize,
                                    * even when referencing into Dictionary content
                                    * default : 0 when using a CDict, 1 when using a Prefix */
-            cctx->forceWindow = value>0;
-            cctx->loadedDictEnd = 0;
-            return 0;
+        cctx->forceWindow = value>0;
+        cctx->loadedDictEnd = 0;
+        return 0;
+
+    case ZSTD_p_nbThreads:
+        if (value==0) return 0;
+#ifndef ZSTD_MULTITHREAD
+        if (value > 1) return ERROR(compressionParameter_unsupported);
+#endif
+        if ((value>1) && (cctx->nbThreads != value)) {
+            ZSTDMT_freeCCtx(cctx->mtctx);
+            cctx->nbThreads = value;
+            cctx->mtctx = ZSTDMT_createCCtx(value);
+            if (cctx->mtctx == NULL) return ERROR(memory_allocation);
+        }
+        cctx->nbThreads = 1;
+        return 0;
+
+    case ZSTDMT_p_jobSize:
+        if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
+        assert(cctx->mtctx != NULL);
+        return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_sectionSize, value);
+
+    case ZSTDMT_p_overlapSizeLog:
+        if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
+        assert(cctx->mtctx != NULL);
+        return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
 
     case ZSTD_p_rawContentDict :  /* load dictionary in "content-only" mode (no header analysis) (default:0) */
         cctx->forceRawDict = value>0;
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 57c0360b0..0d6014608 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -439,7 +439,7 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter,
         mtctx->sectionSize = value;
         return 0;
     case ZSTDMT_p_overlapSectionLog :
-    DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
+        DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
         mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value;
         return 0;
     default :
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index ff7f21687..336215995 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -33,7 +33,7 @@ ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
 
 ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
 ZSTDLIB_API size_t ZSTDMT_estimateCCtxSize(ZSTD_compressionParameters cParams,
-                                           unsigned nbThreads);
+                                           unsigned nbThreads);   /* not ready yet */
 
 
 /* ===   Simple buffer-to-butter one-pass function   === */
diff --git a/lib/zstd.h b/lib/zstd.h
index 1fc35fc55..c71ed493e 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -491,7 +491,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  *  These functions make it possible to estimate memory usage
  *  of a future target object, before its allocation,
  *  given a set of parameters, which vary depending on target object.
- *  The objective is to guide decision before allocation. */
+ *  The objective is to guide decision before allocation.
+ *  Note : CCtx estimation is only correct for single-threaded compression */
 ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
 ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
 
@@ -695,17 +696,17 @@ typedef enum {
     ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
                              /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
 
-#if 0
-    /* multi-threading parameters (not ready yet !) */
+    /* multi-threading parameters */
     ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
-                              * More threads improve speed, but increases also memory usage */
-    ZSTDMT_p_jobSize,        /* Size of a compression job. Each job is compressed in parallel.
+                              * More threads improve speed, but also increase memory usage.
+                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
+                              * Special: value 0 means "do not change nbThreads" */
+    ZSTDMT_p_jobSize,        /* Size of a compression job. Each compression job is completed in parallel.
                               * 0 means default, which is dynamically determined based on compression parameters.
                               * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
                               * The minimum size is automatically and transparently enforced */
     ZSTDMT_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
                               * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-#endif
 
     /* advanced parameters - may not remain available after API update */
     ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,

From 6056e4c3eb55ab7c5000818a4457241b9ccb2019 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 2 Jun 2017 11:36:47 -0700
Subject: [PATCH 045/109] added POOL_sizeof() for single-thread

---
 lib/common/pool.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/common/pool.c b/lib/common/pool.c
index 05adcf1ce..749fa4f2f 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -198,4 +198,9 @@ void POOL_add(void *ctx, POOL_function function, void *opaque) {
   function(opaque);
 }
 
+size_t POOL_sizeof(POOL_ctx *ctx) {
+    if (ctx==NULL) return 0;  /* supports sizeof NULL */
+    return sizeof(*ctx);
+}
+
 #endif  /* ZSTD_MULTITHREAD */

From b877e834b1f349f6861baaf237b38a4547c80ace Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 2 Jun 2017 13:47:11 -0700
Subject: [PATCH 046/109] minor indent

---
 lib/compress/zstdmt_compress.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 336215995..adbb7dfe3 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -32,16 +32,15 @@ ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads,
 ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
 
 ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
-ZSTDLIB_API size_t ZSTDMT_estimateCCtxSize(ZSTD_compressionParameters cParams,
-                                           unsigned nbThreads);   /* not ready yet */
+ZSTDLIB_API size_t ZSTDMT_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned nbThreads);   /* not ready yet */
 
 
 /* ===   Simple buffer-to-butter one-pass function   === */
 
 ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
-                           void* dst, size_t dstCapacity,
-                     const void* src, size_t srcSize,
-                           int compressionLevel);
+                                       void* dst, size_t dstCapacity,
+                                 const void* src, size_t srcSize,
+                                       int compressionLevel);
 
 
 /* ===   Streaming functions   === */

From dcb7535352df45c52149ea6d82b5569a08e005ae Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 2 Jun 2017 14:01:21 -0700
Subject: [PATCH 047/109] ensure zlibwrapper uses ZSTD_malloc() and ZSTD_free()

which is compatible with { NULL, NULL, NULL }
---
 lib/common/zstd_common.c       |  2 +-
 zlibWrapper/zstd_zlibwrapper.c | 54 ++++++++++++++++------------------
 2 files changed, 27 insertions(+), 29 deletions(-)

diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 801483960..93187d5c0 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -61,7 +61,7 @@ void* ZSTD_calloc(size_t size, ZSTD_customMem customMem)
 {
     if (customMem.customAlloc) {
         /* calloc implemented as malloc+memset;
-         * not as efficient, but our best guess for custom malloc */
+         * not as efficient as calloc, but next best guess for custom malloc */
         void* const ptr = customMem.customAlloc(customMem.opaque, size);
         memset(ptr, 0, size);
         return ptr;
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index a078cf556..7a0b09db6 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -17,7 +17,7 @@
 #include "zstd_zlibwrapper.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_MAGICNUMBER */
 #include "zstd.h"
-#include "zstd_internal.h"         /* defaultCustomMem */
+#include "zstd_internal.h"         /* ZSTD_malloc, ZSTD_free */
 
 
 #define Z_INFLATE_SYNC              8
@@ -100,7 +100,7 @@ size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
 {
     if (zwc==NULL) return 0;   /* support free on NULL */
     if (zwc->zbc) ZSTD_freeCStream(zwc->zbc);
-    zwc->customMem.customFree(zwc->customMem.opaque, zwc);
+    ZSTD_free(zwc, zwc->customMem);
     return 0;
 }
 
@@ -472,8 +472,8 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
 {
     if (zwd==NULL) return 0;   /* support free on null */
     if (zwd->zbd) ZSTD_freeDStream(zwd->zbd);
-    if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version);
-    zwd->customMem.customFree(zwd->customMem.opaque, zwd);
+    ZSTD_free(zwd->version, zwd->customMem);
+    ZSTD_free(zwd, zwd->customMem);
     return 0;
 }
 
@@ -505,22 +505,21 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
         return inflateInit(strm);
     }
 
-    {
-    ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm);
-    LOG_WRAPPERD("- inflateInit\n");
-    if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
+    {   ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm);
+        LOG_WRAPPERD("- inflateInit\n");
+        if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
 
-    zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1);
-    if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
-    strcpy(zwd->version, version);
+        zwd->version = ZSTD_malloc(strlen(version)+1, zwd->customMem);
+        if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
+        strcpy(zwd->version, version);
 
-    zwd->stream_size = stream_size;
-    zwd->totalInBytes = 0;
-    strm->state = (struct internal_state*) zwd; /* use state which in not used by user */
-    strm->total_in = 0;
-    strm->total_out = 0;
-    strm->reserved = ZWRAP_UNKNOWN_STREAM; /* mark as unknown steam */
-    strm->adler = 0;
+        zwd->stream_size = stream_size;
+        zwd->totalInBytes = 0;
+        strm->state = (struct internal_state*) zwd; /* use state which in not used by user */
+        strm->total_in = 0;
+        strm->total_out = 0;
+        strm->reserved = ZWRAP_UNKNOWN_STREAM; /* mark as unknown steam */
+        strm->adler = 0;
     }
 
     return Z_OK;
@@ -534,15 +533,14 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int  windowBits,
         return inflateInit2_(strm, windowBits, version, stream_size);
     }
 
-    {
-    int ret = z_inflateInit_ (strm, version, stream_size);
-    LOG_WRAPPERD("- inflateInit2 windowBits=%d\n", windowBits);
-    if (ret == Z_OK) {
-        ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)strm->state;
-        if (zwd == NULL) return Z_STREAM_ERROR;
-        zwd->windowBits = windowBits;
-    }
-    return ret;
+    {   int ret = z_inflateInit_ (strm, version, stream_size);
+        LOG_WRAPPERD("- inflateInit2 windowBits=%d\n", windowBits);
+        if (ret == Z_OK) {
+            ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*)strm->state;
+            if (zwd == NULL) return Z_STREAM_ERROR;
+            zwd->windowBits = windowBits;
+        }
+        return ret;
     }
 }
 
@@ -552,7 +550,7 @@ int ZWRAP_inflateReset_keepDict(z_streamp strm)
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
         return inflateReset(strm);
 
-    {   ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+    {   ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
         if (zwd == NULL) return Z_STREAM_ERROR;
         ZWRAP_initDCtx(zwd);
         zwd->decompState = ZWRAP_useReset;

From 4effccbf56588582ff7963f023018c9f82bcbc07 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 2 Jun 2017 14:24:58 -0700
Subject: [PATCH 048/109] zlib_wrapper's uncompress() uses ZSTD_isFrame() for
 routing

more generic and safer than using own routing for magic number comparison
---
 lib/common/zstd_common.c         |  2 +-
 lib/decompress/zstd_decompress.c | 16 +++++++---------
 zlibWrapper/zstd_zlibwrapper.c   | 16 ++++++++--------
 3 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 93187d5c0..f68167238 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -22,7 +22,7 @@
 /*-****************************************
 *  Version
 ******************************************/
-unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
+unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }
 
 const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 7dfcddb1b..766ff0ff6 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -204,16 +204,14 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
 
 ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
 {
-    ZSTD_DCtx* dctx;
+    if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
 
-    if (!customMem.customAlloc ^ !customMem.customFree)
-        return NULL;
-
-    dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
-    if (!dctx) return NULL;
-    memcpy(&dctx->customMem, &customMem, sizeof(customMem));
-    ZSTD_initDCtx_internal(dctx);
-    return dctx;
+    {   ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem);
+        if (!dctx) return NULL;
+        dctx->customMem = customMem;
+        ZSTD_initDCtx_internal(dctx);
+        return dctx;
+    }
 }
 
 ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 7a0b09db6..ed68a8a07 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -15,7 +15,7 @@
 #define ZLIB_CONST
 #include                   /* without #define Z_PREFIX */
 #include "zstd_zlibwrapper.h"
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_MAGICNUMBER */
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_isFrame, ZSTD_MAGICNUMBER */
 #include "zstd.h"
 #include "zstd_internal.h"         /* ZSTD_malloc, ZSTD_free */
 
@@ -1004,9 +1004,9 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest,   uLongf *destLen,
         return compress2(dest, destLen, source, sourceLen, level);
 
     { size_t dstCapacity = *destLen;
-      size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
-      if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
-      *destLen = errorCode;
+      size_t const cSize = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
+      if (ZSTD_isError(cSize)) return Z_STREAM_ERROR;
+      *destLen = cSize;
     }
     return Z_OK;
 }
@@ -1024,13 +1024,13 @@ ZEXTERN uLong ZEXPORT z_compressBound OF((uLong sourceLen))
 ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest,   uLongf *destLen,
                                    const Bytef *source, uLong sourceLen))
 {
-    if (sourceLen < 4 || MEM_readLE32(source) != ZSTD_MAGICNUMBER)
+    if (!ZSTD_isFrame(source, sourceLen))
         return uncompress(dest, destLen, source, sourceLen);
 
     { size_t dstCapacity = *destLen;
-      size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
-      if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
-      *destLen = errorCode;
+      size_t const dSize = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
+      if (ZSTD_isError(dSize)) return Z_STREAM_ERROR;
+      *destLen = dSize;
      }
     return Z_OK;
 }

From 33a7e679e501005658c8a8ff81fae5d24f4d9eaa Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 2 Jun 2017 17:10:49 -0700
Subject: [PATCH 049/109] significant zlib wrapper code refactoring

code indentation
variable scope and names
constify

Only coding style changes.
The logic should remain the same.
---
 lib/decompress/zstd_decompress.c |   2 +-
 zlibWrapper/Makefile             |  15 +-
 zlibWrapper/zstd_zlibwrapper.c   | 558 ++++++++++++++++---------------
 3 files changed, 305 insertions(+), 270 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 766ff0ff6..33a905a20 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1659,7 +1659,7 @@ size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const
 
 size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
-#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
+#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
     size_t regenSize;
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();
     if (dctx==NULL) return ERROR(memory_allocation);
diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index 5a6378767..4e8fb4a32 100644
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -18,9 +18,12 @@ EXAMPLE_PATH = examples
 PROGRAMS_PATH = ../programs
 TEST_FILE = ../doc/zstd_compression_format.md
 
-CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
+CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH)       \
+           -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
 CFLAGS  ?= $(MOREFLAGS) -O3 -std=gnu99
-CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wstrict-aliasing=1
+CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
+           -Wdeclaration-after-statement -Wstrict-prototypes -Wundef     \
+           -Wstrict-aliasing=1
 
 
 # Define *.exe as extension for Windows systems
@@ -48,8 +51,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench minigzip minigzip_zstd
 	#cp example$(EXT).gz example$(EXT)_gz.gz
 	./minigzip_zstd -d example$(EXT).gz
 	@echo ---- minigzip end ----
-	./zwrapbench -qb3B1K $(TEST_FILE)
-	./zwrapbench -rqb1e5 ../lib ../programs ../tests
+	./zwrapbench -qi1b3B1K $(TEST_FILE)
+	./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
 
 #valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
 valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
@@ -61,8 +64,8 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
 	$(VALGRIND) ./fitblk 40960 <$(TEST_FILE)
 	$(VALGRIND) ./fitblk_zstd 10240 <$(TEST_FILE)
 	$(VALGRIND) ./fitblk_zstd 40960 <$(TEST_FILE)
-	$(VALGRIND) ./zwrapbench -qb3B1K $(TEST_FILE)
-	$(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests
+	$(VALGRIND) ./zwrapbench -qi1b3B1K $(TEST_FILE)
+	$(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
 
 #.c.o:
 #	$(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index ed68a8a07..ade3b88cd 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -8,6 +8,13 @@
  */
 
 
+/* ===   Tuning parameters   === */
+#ifndef ZWRAP_USE_ZSTD
+    #define ZWRAP_USE_ZSTD 0
+#endif
+
+
+/* ===   Dependencies   === */
 #include 
 #include                  /* vsprintf */
 #include                 /* va_list, for z_gzprintf */
@@ -20,24 +27,23 @@
 #include "zstd_internal.h"         /* ZSTD_malloc, ZSTD_free */
 
 
+/* ===   Constants   === */
 #define Z_INFLATE_SYNC              8
 #define ZLIB_HEADERSIZE             4
 #define ZSTD_HEADERSIZE             ZSTD_frameHeaderSize_min
 #define ZWRAP_DEFAULT_CLEVEL        3   /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
 
-#define LOG_WRAPPERC(...)  /* printf(__VA_ARGS__) */
-#define LOG_WRAPPERD(...)  /* printf(__VA_ARGS__) */
+
+/* ===   Debug   === */
+#define LOG_WRAPPERC(...)  /* fprintf(stderr, __VA_ARGS__) */
+#define LOG_WRAPPERD(...)  /* fprintf(stderr, __VA_ARGS__) */
 
 #define FINISH_WITH_GZ_ERR(msg) { (void)msg; return Z_STREAM_ERROR; }
 #define FINISH_WITH_NULL_ERR(msg) { (void)msg; return NULL; }
 
 
-
-#ifndef ZWRAP_USE_ZSTD
-    #define ZWRAP_USE_ZSTD 0
-#endif
-
-static int g_ZWRAP_useZSTDcompression = ZWRAP_USE_ZSTD;   /* 0 = don't use ZSTD */
+/* ===   Wrapper   === */
+static int g_ZWRAP_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */
 
 void ZWRAP_useZSTDcompression(int turn_on) { g_ZWRAP_useZSTDcompression = turn_on; }
 
@@ -63,7 +69,7 @@ static void* ZWRAP_allocFunction(void* opaque, size_t size)
 {
     z_streamp strm = (z_streamp) opaque;
     void* address = strm->zalloc(strm->opaque, 1, (uInt)size);
-  /*  printf("ZWRAP alloc %p, %d \n", address, (int)size); */
+    /* LOG_WRAPPERC("ZWRAP alloc %p, %d \n", address, (int)size); */
     return address;
 }
 
@@ -71,12 +77,12 @@ static void ZWRAP_freeFunction(void* opaque, void* address)
 {
     z_streamp strm = (z_streamp) opaque;
     strm->zfree(strm->opaque, address);
-   /* if (address) printf("ZWRAP free %p \n", address); */
+   /* if (address) LOG_WRAPPERC("ZWRAP free %p \n", address); */
 }
 
 
 
-/* *** Compression *** */
+/* ===   Compression   === */
 typedef enum { ZWRAP_useInit, ZWRAP_useReset, ZWRAP_streamEnd } ZWRAP_state_t;
 
 typedef struct {
@@ -96,16 +102,16 @@ typedef ZWRAP_CCtx internal_state;
 
 
 
-size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
+static size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
 {
     if (zwc==NULL) return 0;   /* support free on NULL */
-    if (zwc->zbc) ZSTD_freeCStream(zwc->zbc);
+    ZSTD_freeCStream(zwc->zbc);
     ZSTD_free(zwc, zwc->customMem);
     return 0;
 }
 
 
-ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
+static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
 {
     ZWRAP_CCtx* zwc;
 
@@ -114,9 +120,8 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
         if (zwc==NULL) return NULL;
         memset(zwc, 0, sizeof(ZWRAP_CCtx));
         memcpy(&zwc->allocFunc, strm, sizeof(z_stream));
-        { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc };
-          memcpy(&zwc->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
-        }
+        { ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc };
+          zwc->customMem = ZWRAP_customMem; }
     } else {
         zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc));
         if (zwc==NULL) return NULL;
@@ -126,23 +131,25 @@ ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
 }
 
 
-int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dictSize, unsigned long long pledgedSrcSize)
+static int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dictSize, unsigned long long pledgedSrcSize)
 {
     LOG_WRAPPERC("- ZWRAP_initializeCStream=%p\n", zwc);
     if (zwc == NULL || zwc->zbc == NULL) return Z_STREAM_ERROR;
 
     if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize;
-    { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize);
-      size_t errorCode;
-      LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy);
-      errorCode = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize);
-      if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; }
+    {   ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize);
+        size_t initErr;
+        LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n",
+                    (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy);
+        initErr = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize);
+        if (ZSTD_isError(initErr)) return Z_STREAM_ERROR;
+    }
 
     return Z_OK;
 }
 
 
-int ZWRAPC_finishWithError(ZWRAP_CCtx* zwc, z_streamp strm, int error)
+static int ZWRAPC_finishWithError(ZWRAP_CCtx* zwc, z_streamp strm, int error)
 {
     LOG_WRAPPERC("- ZWRAPC_finishWithError=%d\n", error);
     if (zwc) ZWRAP_freeCCtx(zwc);
@@ -151,7 +158,7 @@ int ZWRAPC_finishWithError(ZWRAP_CCtx* zwc, z_streamp strm, int error)
 }
 
 
-int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message)
+static int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message)
 {
     ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
     strm->msg = message;
@@ -276,34 +283,36 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
     ZWRAP_CCtx* zwc;
 
     if (!g_ZWRAP_useZSTDcompression) {
-        int res;
-        LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
-        res = deflate(strm, flush);
-        return res;
+        LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n",
+                    (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+        return deflate(strm, flush);
     }
 
     zwc = (ZWRAP_CCtx*) strm->state;
     if (zwc == NULL) { LOG_WRAPPERC("zwc == NULL\n"); return Z_STREAM_ERROR; }
 
     if (zwc->zbc == NULL) {
-        int res;
         zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem);
         if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0);
-        res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
-        if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
+        { int const initErr = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
+          if (initErr != Z_OK) return ZWRAPC_finishWithError(zwc, strm, initErr); }
         if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
     } else {
         if (zwc->totalInBytes == 0) {
             if (zwc->comprState == ZWRAP_useReset) {
-                size_t const errorCode = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize);
-                if (ZSTD_isError(errorCode)) { LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); return ZWRAPC_finishWithError(zwc, strm, 0); }
+                size_t const resetErr = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize);
+                if (ZSTD_isError(resetErr)) {
+                    LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n",
+                                ZSTD_getErrorName(resetErr));
+                    return ZWRAPC_finishWithError(zwc, strm, 0);
+                }
             } else {
-                int res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
+                int const res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0);
                 if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
                 if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset;
             }
-        }
-    }
+        }  /* (zwc->totalInBytes == 0) */
+    }  /* ! (zwc->zbc == NULL) */
 
     LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
     if (strm->avail_in > 0) {
@@ -313,9 +322,9 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
         zwc->outBuffer.dst = strm->next_out;
         zwc->outBuffer.size = strm->avail_out;
         zwc->outBuffer.pos = 0;
-        { size_t const errorCode = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer);
+        { size_t const cErr = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer);
           LOG_WRAPPERC("deflate ZSTD_compressStream srcSize=%d dstCapacity=%d\n", (int)zwc->inBuffer.size, (int)zwc->outBuffer.size);
-          if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0);
+          if (ZSTD_isError(cErr)) return ZWRAPC_finishWithError(zwc, strm, 0);
         }
         strm->next_out += zwc->outBuffer.pos;
         strm->total_out += zwc->outBuffer.pos;
@@ -345,8 +354,12 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
         strm->next_out += zwc->outBuffer.pos;
         strm->total_out += zwc->outBuffer.pos;
         strm->avail_out -= zwc->outBuffer.pos;
-        if (bytesLeft == 0) { zwc->streamEnd = 1; LOG_WRAPPERC("Z_STREAM_END2 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n", (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out); return Z_STREAM_END; }
-    }
+        if (bytesLeft == 0) {
+            zwc->streamEnd = 1;
+            LOG_WRAPPERC("Z_STREAM_END2 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n",
+                        (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out);
+            return Z_STREAM_END;
+    }   }
     else
     if (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH) {
         size_t bytesLeft;
@@ -409,12 +422,13 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm,
 
 
 
-/* *** Decompression *** */
+/* ===   Decompression   === */
+
 typedef enum { ZWRAP_ZLIB_STREAM, ZWRAP_ZSTD_STREAM, ZWRAP_UNKNOWN_STREAM } ZWRAP_stream_type;
 
 typedef struct {
     ZSTD_DStream* zbd;
-    char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */
+    char headerBuf[16];   /* must be >= ZSTD_frameHeaderSize_min */
     int errorCount;
     unsigned long long totalInBytes; /* we need it as strm->total_in can be reset by user */
     ZWRAP_state_t decompState;
@@ -426,10 +440,48 @@ typedef struct {
     char *version;
     int windowBits;
     ZSTD_customMem customMem;
-    z_stream allocFunc; /* copy of zalloc, zfree, opaque */
+    z_stream allocFunc; /* just to copy zalloc, zfree, opaque */
 } ZWRAP_DCtx;
 
 
+static void ZWRAP_initDCtx(ZWRAP_DCtx* zwd)
+{
+    zwd->errorCount = 0;
+    zwd->outBuffer.pos = 0;
+    zwd->outBuffer.size = 0;
+}
+
+static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
+{
+    ZWRAP_DCtx* zwd;
+    MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN);   /* check static buffer size condition */
+
+    if (strm->zalloc && strm->zfree) {
+        zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));
+        if (zwd==NULL) return NULL;
+        memset(zwd, 0, sizeof(ZWRAP_DCtx));
+        zwd->allocFunc = *strm;  /* just to copy zalloc, zfree & opaque */
+        { ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc };
+          zwd->customMem = ZWRAP_customMem; }
+    } else {
+        zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd));
+        if (zwd==NULL) return NULL;
+    }
+
+    ZWRAP_initDCtx(zwd);
+    return zwd;
+}
+
+static size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
+{
+    if (zwd==NULL) return 0;   /* support free on null */
+    ZSTD_freeDStream(zwd->zbd);
+    ZSTD_free(zwd->version, zwd->customMem);
+    ZSTD_free(zwd, zwd->customMem);
+    return 0;
+}
+
+
 int ZWRAP_isUsingZSTDdecompression(z_streamp strm)
 {
     if (strm == NULL) return 0;
@@ -437,59 +489,17 @@ int ZWRAP_isUsingZSTDdecompression(z_streamp strm)
 }
 
 
-void ZWRAP_initDCtx(ZWRAP_DCtx* zwd)
-{
-    zwd->errorCount = 0;
-    zwd->outBuffer.pos = 0;
-    zwd->outBuffer.size = 0;
-}
-
-
-ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
-{
-    ZWRAP_DCtx* zwd;
-
-    if (strm->zalloc && strm->zfree) {
-        zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));
-        if (zwd==NULL) return NULL;
-        memset(zwd, 0, sizeof(ZWRAP_DCtx));
-        memcpy(&zwd->allocFunc, strm, sizeof(z_stream));
-        { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc };
-          memcpy(&zwd->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
-        }
-    } else {
-        zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd));
-        if (zwd==NULL) return NULL;
-    }
-
-    MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN);   /* if compilation fails here, assertion is false */
-    ZWRAP_initDCtx(zwd);
-    return zwd;
-}
-
-
-size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
-{
-    if (zwd==NULL) return 0;   /* support free on null */
-    if (zwd->zbd) ZSTD_freeDStream(zwd->zbd);
-    ZSTD_free(zwd->version, zwd->customMem);
-    ZSTD_free(zwd, zwd->customMem);
-    return 0;
-}
-
-
-int ZWRAPD_finishWithError(ZWRAP_DCtx* zwd, z_streamp strm, int error)
+static int ZWRAPD_finishWithError(ZWRAP_DCtx* zwd, z_streamp strm, int error)
 {
     LOG_WRAPPERD("- ZWRAPD_finishWithError=%d\n", error);
-    if (zwd) ZWRAP_freeDCtx(zwd);
-    if (strm) strm->state = NULL;
+    ZWRAP_freeDCtx(zwd);
+    strm->state = NULL;
     return (error) ? error : Z_STREAM_ERROR;
 }
 
-
-int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
+static int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
 {
-    ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+    ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
     strm->msg = message;
     if (zwd == NULL) return Z_STREAM_ERROR;
 
@@ -501,11 +511,11 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
                                      const char *version, int stream_size))
 {
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
-        strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
+        strm->reserved = ZWRAP_ZLIB_STREAM;
         return inflateInit(strm);
     }
 
-    {   ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm);
+    {   ZWRAP_DCtx* const zwd = ZWRAP_createDCtx(strm);
         LOG_WRAPPERD("- inflateInit\n");
         if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
 
@@ -515,10 +525,10 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
 
         zwd->stream_size = stream_size;
         zwd->totalInBytes = 0;
-        strm->state = (struct internal_state*) zwd; /* use state which in not used by user */
+        strm->state = (struct internal_state*) zwd;
         strm->total_in = 0;
         strm->total_out = 0;
-        strm->reserved = ZWRAP_UNKNOWN_STREAM; /* mark as unknown steam */
+        strm->reserved = ZWRAP_UNKNOWN_STREAM;
         strm->adler = 0;
     }
 
@@ -533,7 +543,7 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int  windowBits,
         return inflateInit2_(strm, windowBits, version, stream_size);
     }
 
-    {   int ret = z_inflateInit_ (strm, version, stream_size);
+    {   int const ret = z_inflateInit_ (strm, version, stream_size);
         LOG_WRAPPERD("- inflateInit2 windowBits=%d\n", windowBits);
         if (ret == Z_OK) {
             ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*)strm->state;
@@ -569,10 +579,10 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
         return inflateReset(strm);
 
-    { int ret = ZWRAP_inflateReset_keepDict(strm);
+    { int const ret = ZWRAP_inflateReset_keepDict(strm);
       if (ret != Z_OK) return ret; }
 
-    { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+    { ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
       if (zwd == NULL) return Z_STREAM_ERROR;
       zwd->decompState = ZWRAP_useInit; }
 
@@ -587,9 +597,9 @@ ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm,
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
         return inflateReset2(strm, windowBits);
 
-    {   int ret = z_inflateReset (strm);
+    {   int const ret = z_inflateReset (strm);
         if (ret == Z_OK) {
-            ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)strm->state;
+            ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*)strm->state;
             if (zwd == NULL) return Z_STREAM_ERROR;
             zwd->windowBits = windowBits;
         }
@@ -607,11 +617,10 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
         return inflateSetDictionary(strm, dictionary, dictLength);
 
-    {   size_t errorCode;
-        ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+    {   ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
         if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
-        errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
-        if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
+        { size_t const initErr = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
+          if (ZSTD_isError(initErr)) return ZWRAPD_finishWithError(zwd, strm, 0); }
         zwd->decompState = ZWRAP_useReset;
 
         if (zwd->totalInBytes == ZSTD_HEADERSIZE) {
@@ -621,14 +630,14 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
             zwd->outBuffer.dst = strm->next_out;
             zwd->outBuffer.size = 0;
             zwd->outBuffer.pos = 0;
-            errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
-            LOG_WRAPPERD("inflateSetDictionary ZSTD_decompressStream errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
-            if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) {
-                LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode));
-                return ZWRAPD_finishWithError(zwd, strm, 0);
-            }
-        }
-    }
+            {   size_t const errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
+                LOG_WRAPPERD("inflateSetDictionary ZSTD_decompressStream errorCode=%d srcSize=%d dstCapacity=%d\n",
+                             (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
+                if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) {
+                    LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n",
+                                 ZSTD_getErrorName(errorCode));
+                    return ZWRAPD_finishWithError(zwd, strm, 0);
+    }   }   }   }
 
     return Z_OK;
 }
@@ -637,156 +646,175 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
 ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
 {
     ZWRAP_DCtx* zwd;
-    int res;
+
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) {
-        LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
-        res = inflate(strm, flush);
-        LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
-        return res;
+        int const result = inflate(strm, flush);
+        LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n",
+                     (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, result);
+        return result;
     }
 
     if (strm->avail_in <= 0) return Z_OK;
 
-    {   size_t errorCode, srcSize;
-        zwd = (ZWRAP_DCtx*) strm->state;
-        LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+    zwd = (ZWRAP_DCtx*) strm->state;
+    LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n",
+                 (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
 
-        if (zwd == NULL) return Z_STREAM_ERROR;
-        if (zwd->decompState == ZWRAP_streamEnd) return Z_STREAM_END;
+    if (zwd == NULL) return Z_STREAM_ERROR;
+    if (zwd->decompState == ZWRAP_streamEnd) return Z_STREAM_END;
 
-        if (zwd->totalInBytes < ZLIB_HEADERSIZE) {
-            if (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
-                if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
-                    if (zwd->windowBits)
-                        errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
-                    else
-                        errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
-
-                    strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
-                    errorCode = ZWRAP_freeDCtx(zwd);
-                    if (ZSTD_isError(errorCode)) goto error;
-
-                    if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
-                    else res = inflate(strm, flush);
-                    LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
-                    return res;
-                }
-            } else {
-                srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - zwd->totalInBytes);
-                memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
-                strm->total_in += srcSize;
-                zwd->totalInBytes += srcSize;
-                strm->next_in += srcSize;
-                strm->avail_in -= srcSize;
-                if (zwd->totalInBytes < ZLIB_HEADERSIZE) return Z_OK;
-
-                if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
-                    z_stream strm2;
-                    strm2.next_in = strm->next_in;
-                    strm2.avail_in = strm->avail_in;
-                    strm2.next_out = strm->next_out;
-                    strm2.avail_out = strm->avail_out;
-
-                    if (zwd->windowBits)
-                        errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
-                    else
-                        errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
-                    LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
-                    if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
-
-                    /* inflate header */
-                    strm->next_in = (unsigned char*)zwd->headerBuf;
-                    strm->avail_in = ZLIB_HEADERSIZE;
-                    strm->avail_out = 0;
-                    errorCode = inflate(strm, Z_NO_FLUSH);
-                    LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
-                    if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
-                    if (strm->avail_in > 0) goto error;
-
-                    strm->next_in = strm2.next_in;
-                    strm->avail_in = strm2.avail_in;
-                    strm->next_out = strm2.next_out;
-                    strm->avail_out = strm2.avail_out;
-
-                    strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
-                    errorCode = ZWRAP_freeDCtx(zwd);
-                    if (ZSTD_isError(errorCode)) goto error;
-
-                    if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
-                    else res = inflate(strm, flush);
-                    LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
-                    return res;
-                }
-            }
-        }
-
-        strm->reserved = ZWRAP_ZSTD_STREAM; /* mark as zstd steam */
-
-        if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
-
-        if (!zwd->zbd) {
-            zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
-            if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
-            zwd->decompState = ZWRAP_useInit;
-        }
-
-        if (zwd->totalInBytes < ZSTD_HEADERSIZE)
-        {
-            if (zwd->totalInBytes == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
-                if (zwd->decompState == ZWRAP_useInit) {
-                    errorCode = ZSTD_initDStream(zwd->zbd);
-                    if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
-                } else {
-                    errorCode = ZSTD_resetDStream(zwd->zbd);
-                    if (ZSTD_isError(errorCode)) goto error;
-                }
-            } else {
-                srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - zwd->totalInBytes);
-                memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
-                strm->total_in += srcSize;
-                zwd->totalInBytes += srcSize;
-                strm->next_in += srcSize;
-                strm->avail_in -= srcSize;
-                if (zwd->totalInBytes < ZSTD_HEADERSIZE) return Z_OK;
-
-                if (zwd->decompState == ZWRAP_useInit) {
-                    errorCode = ZSTD_initDStream(zwd->zbd);
-                    if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
-                } else {
-                    errorCode = ZSTD_resetDStream(zwd->zbd);
-                    if (ZSTD_isError(errorCode)) goto error;
+    if (zwd->totalInBytes < ZLIB_HEADERSIZE) {
+        if (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
+            if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
+                {   int const initErr = (zwd->windowBits) ?
+                                inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size) :
+                                inflateInit_(strm, zwd->version, zwd->stream_size);
+                    LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", initErr);
+                    if (initErr != Z_OK) return ZWRAPD_finishWithError(zwd, strm, initErr);
                 }
 
-                zwd->inBuffer.src = zwd->headerBuf;
-                zwd->inBuffer.size = ZSTD_HEADERSIZE;
-                zwd->inBuffer.pos = 0;
-                zwd->outBuffer.dst = strm->next_out;
-                zwd->outBuffer.size = 0;
-                zwd->outBuffer.pos = 0;
-                errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
-                LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
-                if (ZSTD_isError(errorCode)) {
-                    LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
+                strm->reserved = ZWRAP_ZLIB_STREAM;
+                { size_t const freeErr = ZWRAP_freeDCtx(zwd);
+                  if (ZSTD_isError(freeErr)) goto error; }
+
+                {   int const result = (flush == Z_INFLATE_SYNC) ?
+                                        inflateSync(strm) :
+                                        inflate(strm, flush);
+                    LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n",
+                                 (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
+                    return result;
+            }   }
+        } else {  /* ! (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) */
+            size_t const srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - zwd->totalInBytes);
+            memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
+            strm->total_in += srcSize;
+            zwd->totalInBytes += srcSize;
+            strm->next_in += srcSize;
+            strm->avail_in -= srcSize;
+            if (zwd->totalInBytes < ZLIB_HEADERSIZE) return Z_OK;
+
+            if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
+                z_stream strm2;
+                strm2.next_in = strm->next_in;
+                strm2.avail_in = strm->avail_in;
+                strm2.next_out = strm->next_out;
+                strm2.avail_out = strm->avail_out;
+
+                {   int const initErr = (zwd->windowBits) ?
+                                inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size) :
+                                inflateInit_(strm, zwd->version, zwd->stream_size);
+                    LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", initErr);
+                    if (initErr != Z_OK) return ZWRAPD_finishWithError(zwd, strm, initErr);
+                }
+
+                /* inflate header */
+                strm->next_in = (unsigned char*)zwd->headerBuf;
+                strm->avail_in = ZLIB_HEADERSIZE;
+                strm->avail_out = 0;
+                {   int const dErr = inflate(strm, Z_NO_FLUSH);
+                    LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n",
+                                  dErr, (int)strm->avail_in);
+                    if (dErr != Z_OK)
+                        return ZWRAPD_finishWithError(zwd, strm, dErr);
+                }
+                if (strm->avail_in > 0) goto error;
+
+                strm->next_in = strm2.next_in;
+                strm->avail_in = strm2.avail_in;
+                strm->next_out = strm2.next_out;
+                strm->avail_out = strm2.avail_out;
+
+                strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */
+                { size_t const freeErr = ZWRAP_freeDCtx(zwd);
+                  if (ZSTD_isError(freeErr)) goto error; }
+
+                {   int const result = (flush == Z_INFLATE_SYNC) ?
+                                       inflateSync(strm) :
+                                       inflate(strm, flush);
+                    LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n",
+                                 (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
+                    return result;
+        }   }   }  /* if ! (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) */
+    }  /* (zwd->totalInBytes < ZLIB_HEADERSIZE) */
+
+    strm->reserved = ZWRAP_ZSTD_STREAM; /* mark as zstd steam */
+
+    if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
+
+    if (!zwd->zbd) {
+        zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
+        if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
+        zwd->decompState = ZWRAP_useInit;
+    }
+
+    if (zwd->totalInBytes < ZSTD_HEADERSIZE) {
+        if (zwd->totalInBytes == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
+            if (zwd->decompState == ZWRAP_useInit) {
+                size_t const initErr = ZSTD_initDStream(zwd->zbd);
+                if (ZSTD_isError(initErr)) {
+                    LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n",
+                                 ZSTD_getErrorName(initErr));
                     goto error;
                 }
-                if (zwd->inBuffer.pos != zwd->inBuffer.size) goto error; /* not consumed */
+            } else {
+                size_t const resetErr = ZSTD_resetDStream(zwd->zbd);
+                if (ZSTD_isError(resetErr)) goto error;
             }
-        }
+        } else {
+            size_t const srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - zwd->totalInBytes);
+            memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize);
+            strm->total_in += srcSize;
+            zwd->totalInBytes += srcSize;
+            strm->next_in += srcSize;
+            strm->avail_in -= srcSize;
+            if (zwd->totalInBytes < ZSTD_HEADERSIZE) return Z_OK;
 
-        zwd->inBuffer.src = strm->next_in;
-        zwd->inBuffer.size = strm->avail_in;
-        zwd->inBuffer.pos = 0;
-        zwd->outBuffer.dst = strm->next_out;
-        zwd->outBuffer.size = strm->avail_out;
-        zwd->outBuffer.pos = 0;
-        errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
-        LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out);
-        if (ZSTD_isError(errorCode)) {
+            if (zwd->decompState == ZWRAP_useInit) {
+                size_t const initErr = ZSTD_initDStream(zwd->zbd);
+                if (ZSTD_isError(initErr)) {
+                    LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n",
+                                ZSTD_getErrorName(initErr));
+                    goto error;
+                }
+            } else {
+                size_t const resetErr = ZSTD_resetDStream(zwd->zbd);
+                if (ZSTD_isError(resetErr)) goto error;
+            }
+
+            zwd->inBuffer.src = zwd->headerBuf;
+            zwd->inBuffer.size = ZSTD_HEADERSIZE;
+            zwd->inBuffer.pos = 0;
+            zwd->outBuffer.dst = strm->next_out;
+            zwd->outBuffer.size = 0;
+            zwd->outBuffer.pos = 0;
+            {   size_t const dErr = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
+                LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n",
+                            (int)dErr, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
+                if (ZSTD_isError(dErr)) {
+                    LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(dErr));
+                    goto error;
+            }   }
+            if (zwd->inBuffer.pos != zwd->inBuffer.size) goto error; /* not consumed */
+        }
+    }   /* (zwd->totalInBytes < ZSTD_HEADERSIZE) */
+
+    zwd->inBuffer.src = strm->next_in;
+    zwd->inBuffer.size = strm->avail_in;
+    zwd->inBuffer.pos = 0;
+    zwd->outBuffer.dst = strm->next_out;
+    zwd->outBuffer.size = strm->avail_out;
+    zwd->outBuffer.pos = 0;
+    {   size_t const dErr = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
+        LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n",
+                    (int)dErr, (int)strm->avail_in, (int)strm->avail_out);
+        if (ZSTD_isError(dErr)) {
             zwd->errorCount++;
-            LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n", ZSTD_getErrorName(errorCode), zwd->errorCount);
+            LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n",
+                        ZSTD_getErrorName(dErr), zwd->errorCount);
             if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error;
         }
-        LOG_WRAPPERD("inflate inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size);
+        LOG_WRAPPERD("inflate inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n",
+                    (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size);
         strm->next_out += zwd->outBuffer.pos;
         strm->total_out += zwd->outBuffer.pos;
         strm->avail_out -= zwd->outBuffer.pos;
@@ -794,13 +822,16 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
         zwd->totalInBytes += zwd->inBuffer.pos;
         strm->next_in += zwd->inBuffer.pos;
         strm->avail_in -= zwd->inBuffer.pos;
-        if (errorCode == 0) {
-            LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+        if (dErr == 0) {
+            LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n",
+                        (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
             zwd->decompState = ZWRAP_streamEnd;
             return Z_STREAM_END;
         }
-    }
-    LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
+    }  /* dErr lifetime */
+
+    LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n",
+                (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
     return Z_OK;
 
 error:
@@ -813,13 +844,13 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm))
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
         return inflateEnd(strm);
 
-    LOG_WRAPPERD("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
-    {   size_t errorCode;
-        ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+    LOG_WRAPPERD("- inflateEnd total_in=%d total_out=%d\n",
+                (int)(strm->total_in), (int)(strm->total_out));
+    {   ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state;
         if (zwd == NULL) return Z_OK;  /* structures are already freed */
+        { size_t const freeErr = ZWRAP_freeDCtx(zwd);
+          if (ZSTD_isError(freeErr)) return Z_STREAM_ERROR; }
         strm->state = NULL;
-        errorCode = ZWRAP_freeDCtx(zwd);
-        if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
     }
     return Z_OK;
 }
@@ -836,8 +867,6 @@ ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm))
 
 
 
-
-
 /* Advanced compression functions */
 ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest,
                                     z_streamp source))
@@ -977,7 +1006,7 @@ ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags();
 
 
 
-                        /* utility functions */
+                    /* ===   utility functions  === */
 #ifndef Z_SOLO
 
 ZEXTERN int ZEXPORT z_compress OF((Bytef *dest,   uLongf *destLen,
@@ -986,11 +1015,14 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest,   uLongf *destLen,
     if (!g_ZWRAP_useZSTDcompression)
         return compress(dest, destLen, source, sourceLen);
 
-    { size_t dstCapacity = *destLen;
-      size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL);
-      LOG_WRAPPERD("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity);
-      if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
-      *destLen = errorCode;
+    {   size_t dstCapacity = *destLen;
+        size_t const cSize = ZSTD_compress(dest, dstCapacity,
+                                           source, sourceLen,
+                                           ZWRAP_DEFAULT_CLEVEL);
+        LOG_WRAPPERD("z_compress sourceLen=%d dstCapacity=%d\n",
+                    (int)sourceLen, (int)dstCapacity);
+        if (ZSTD_isError(cSize)) return Z_STREAM_ERROR;
+        *destLen = cSize;
     }
     return Z_OK;
 }

From 8ddf4c22d5e64736aab775990c17e6f7719316b3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 2 Jun 2017 17:16:49 -0700
Subject: [PATCH 050/109] fixed missing initialization

---
 lib/decompress/zstd_decompress.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 33a905a20..68ec13d1a 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -209,6 +209,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
     {   ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem);
         if (!dctx) return NULL;
         dctx->customMem = customMem;
+        dctx->legacyContext = NULL;
         ZSTD_initDCtx_internal(dctx);
         return dctx;
     }

From 257a7226d8ef6049d8aa8e07f1324fbff3947050 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 2 Jun 2017 17:35:11 -0700
Subject: [PATCH 051/109] updated NEWS for v1.3.0

---
 NEWS | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/NEWS b/NEWS
index 7d9c9c94e..ffd1e036f 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,11 @@
+v1.3.0
+cli : changed : `-t *` does no longer stop after a decompression error
+API : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
+API : changed : unified CCtx/CStream and DCtx/DStream definitions
+API exp : changed : slowest strategy renamed ZSTD_btultra
+new : contrib/seekable_format, demo and API, by Sean Purcell
+changed : contrib/linux-kernel, updated version and license, by Nick Terrell
+
 v1.2.0
 cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
 cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell

From 86bd83ef121fe683903dadd8135c7ca12cec4e51 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 2 Jun 2017 17:43:55 -0700
Subject: [PATCH 052/109] completed NEWS for v1.3.0

---
 NEWS | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/NEWS b/NEWS
index ffd1e036f..9f1298126 100644
--- a/NEWS
+++ b/NEWS
@@ -1,8 +1,9 @@
 v1.3.0
 cli : changed : `-t *` does no longer stop after a decompression error
-API : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
-API : changed : unified CCtx/CStream and DCtx/DStream definitions
-API exp : changed : slowest strategy renamed ZSTD_btultra
+API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
+API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
+API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
+API exp : changed : strongest strategy renamed ZSTD_btultra
 new : contrib/seekable_format, demo and API, by Sean Purcell
 changed : contrib/linux-kernel, updated version and license, by Nick Terrell
 

From 58e8d793e18e0884de6072f1bb6dec9b6847cb4f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 2 Jun 2017 18:20:48 -0700
Subject: [PATCH 053/109] made debug definitions common within zstd_internal.h

---
 doc/zstd_manual.html             | 13 ++++++-----
 lib/common/bitstream.h           | 14 ++++++-----
 lib/common/zstd_internal.h       | 31 +++++++++++++++++++++++--
 lib/compress/zstd_compress.c     | 40 ++++++--------------------------
 lib/compress/zstdmt_compress.c   | 27 ++++++++-------------
 lib/compress/zstdmt_compress.h   | 10 ++++----
 lib/decompress/zstd_decompress.c | 13 -----------
 7 files changed, 67 insertions(+), 81 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 3b7f32870..40cae8f28 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -404,7 +404,8 @@ size_t ZSTD_estimateDCtxSize(void);
 
  These functions make it possible to estimate memory usage
   of a future target object, before its allocation,
   given a set of parameters, which vary depending on target object.
-  The objective is to guide decision before allocation. 
+  The objective is to guide decision before allocation.
+  Note : CCtx estimation is only correct for single-threaded compression 
 


 
 
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
@@ -575,17 +576,17 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
     ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
                              /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
 
-#if 0
-    /* multi-threading parameters (not ready yet !) */
+    /* multi-threading parameters */
     ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
-                              * More threads improve speed, but increases also memory usage */
-    ZSTDMT_p_jobSize,        /* Size of a compression job. Each job is compressed in parallel.
+                              * More threads improve speed, but also increase memory usage.
+                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
+                              * Special: value 0 means "do not change nbThreads" */
+    ZSTDMT_p_jobSize,        /* Size of a compression job. Each compression job is completed in parallel.
                               * 0 means default, which is dynamically determined based on compression parameters.
                               * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
                               * The minimum size is automatically and transparently enforced */
     ZSTDMT_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
                               * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-#endif
 
     /* advanced parameters - may not remain available after API update */
     ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index 74eb4b635..f3c9064bb 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -58,7 +58,9 @@ extern "C" {
 #if defined(BIT_DEBUG) && (BIT_DEBUG>=1)
 #  include 
 #else
-#  define assert(condition) ((void)0)
+#  ifndef assert
+#    define assert(condition) ((void)0)
+#  endif
 #endif
 
 
@@ -307,19 +309,19 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
 
 	    case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
 	            /* fall-through */
-	
+
 	    case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
 	            /* fall-through */
-	
+
 	    case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
 	            /* fall-through */
-	
+
 	    case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
 	            /* fall-through */
-	
+
 	    case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
 	            /* fall-through */
-	
+
             default: break;
         }
         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 6f9b4ef1f..8c627cb7a 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -51,9 +51,36 @@
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
 #ifndef XXH_STATIC_LINKING_ONLY
-#  define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
+#  define XXH_STATIC_LINKING_ONLY  /* XXH64_state_t */
+#endif
+#include "xxhash.h"                /* XXH_reset, update, digest */
+
+
+/*-*************************************
+*  Debug
+***************************************/
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
+#  include 
+#else
+#  ifndef assert
+#    define assert(condition) ((void)0)
+#  endif
+#endif
+
+#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
+
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
+#  include 
+   static unsigned g_debugLevel = ZSTD_DEBUG;
+#  define DEBUGLOG(l, ...) {                          \
+                if (l<=g_debugLevel) {                \
+                    fprintf(stderr, __FILE__ ": ");   \
+                    fprintf(stderr, __VA_ARGS__);     \
+                    fprintf(stderr, " \n");           \
+            }   }
+#else
+#  define DEBUGLOG(l, ...)      {}    /* disabled */
 #endif
-#include "xxhash.h"               /* XXH_reset, update, digest */
 
 
 /*-*************************************
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 8f7c200b2..76d3614c7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -28,31 +28,6 @@
 #include "zstdmt_compress.h"
 
 
-/*-*************************************
-*  Debug
-***************************************/
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
-#  include 
-#else
-#  define assert(condition) ((void)0)
-#endif
-
-#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
-
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
-#  include 
-   static unsigned g_debugLevel = ZSTD_DEBUG;
-#  define DEBUGLOG(l, ...) {                          \
-                if (l<=g_debugLevel) {                \
-                    fprintf(stderr, __FILE__ ": ");   \
-                    fprintf(stderr, __VA_ARGS__);     \
-                    fprintf(stderr, " \n");           \
-            }   }
-#else
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
-#endif
-
-
 /*-*************************************
 *  Constants
 ***************************************/
@@ -3622,17 +3597,19 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                                         ZSTD_inBuffer* input,
                                         ZSTD_EndDirective const flushMode)
 {
-    U32 someMoreWork = 1;
     const char* const istart = (const char*)input->src;
     const char* const iend = istart + input->size;
     const char* ip = istart + input->pos;
     char* const ostart = (char*)output->dst;
     char* const oend = ostart + output->size;
     char* op = ostart + output->pos;
+    U32 someMoreWork = 1;
 
-    /* expected to be already allocated */
+    /* check expectations */
     assert(zcs->inBuff != NULL);
     assert(zcs->outBuff!= NULL);
+    assert(output->pos <= output->size);
+    assert(input->pos <= input->size);
 
     while (someMoreWork) {
         switch(zcs->streamStage)
@@ -3765,7 +3742,6 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     if (cctx->streamStage == zcss_init) {
         /* transparent reset */
         ZSTD_parameters params = cctx->requestedParams;
-        DEBUGLOG(5, "ZSTD_compress_generic : transparent reset");
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
                                     cctx->frameContentSize, 0 /* dictSize */);
@@ -3775,7 +3751,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     DEBUGLOG(5, "starting ZSTD_compressStream_generic");
     CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
 
-    DEBUGLOG(5, "completing ZSTD_compress_generic_integral");
+    DEBUGLOG(5, "completing ZSTD_compress_generic");
     return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
 }
 
@@ -3788,12 +3764,10 @@ size_t ZSTD_compress_generic_simpleArgs (
     ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
     ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
     /* ZSTD_compress_generic() will check validity of dstPos and srcPos */
-    size_t const hint = ZSTD_compress_generic(cctx, &output, &input, endOp);
-    if (ZSTD_isError(hint)) return hint;
-
+    size_t const cErr = ZSTD_compress_generic(cctx, &output, &input, endOp);
     *dstPos = output.pos;
     *srcPos = input.pos;
-    return hint;
+    return cErr;
 
 }
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 0d6014608..54feb86f5 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -14,25 +14,19 @@
 
 /* ======   Compiler specifics   ====== */
 #if defined(_MSC_VER)
-#  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
+#  pragma warning(disable : 4204)   /* disable: C4204: non-constant aggregate initializer */
 #endif
 
 
 /* ======   Dependencies   ====== */
-#include      /* memcpy, memset */
-#include "pool.h"       /* threadpool */
-#include "threading.h"  /* mutex */
+#include       /* memcpy, memset */
+#include "pool.h"        /* threadpool */
+#include "threading.h"   /* mutex */
 #include "zstd_internal.h"  /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
 #include "zstdmt_compress.h"
 
 
 /* ======   Debug   ====== */
-#if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=1)
-#  include 
-#else
-#  define assert(condition) ((void)0)
-#endif
-
 #if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=2)
 
 #  include 
@@ -573,17 +567,15 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
 
 
 static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
-                                    const void* dict, size_t dictSize, unsigned updateDict,
-                                    ZSTD_parameters params, unsigned long long pledgedSrcSize)
+                        const void* dict, size_t dictSize, unsigned updateDict,
+                        ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
-    ZSTD_customMem const cmem = { NULL, NULL, NULL };
-    DEBUGLOG(3, "Started new compression, with windowLog : %u",
-                    params.cParams.windowLog);
     if (zcs->nbThreads==1)
         return ZSTD_initCStream_advanced(zcs->cctxPool->cctx[0],
                                         dict, dictSize,
                                         params, pledgedSrcSize);
-    if (zcs->allJobsCompleted == 0) {   /* previous job not correctly finished */
+
+    if (zcs->allJobsCompleted == 0) {   /* previous compression not correctly finished */
         ZSTDMT_waitForAllJobsCompleted(zcs);
         ZSTDMT_releaseAllJobResources(zcs);
         zcs->allJobsCompleted = 1;
@@ -592,7 +584,8 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     if (updateDict) {
         ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
         if (dict && dictSize) {
-            zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0, params.cParams, cmem);
+            zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0 /* byRef */,
+                                                params.cParams, zcs->cMem);
             if (zcs->cdict == NULL) return ERROR(memory_allocation);
     }   }
     zcs->frameContentSize = pledgedSrcSize;
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index adbb7dfe3..ceff32ef6 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -15,8 +15,8 @@
  #endif
 
 
-/* Note : All prototypes defined in this file shall be considered experimental.
- *        There is no guarantee of API continuity (yet) on any of these prototypes */
+/* Note : All prototypes defined in this file must be considered experimental.
+ *        There is no guarantee of API continuity on any of these prototypes */
 
 /* ===   Dependencies   === */
 #include    /* size_t */
@@ -60,8 +60,10 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
 #  define ZSTDMT_SECTION_SIZE_MIN (1U << 20)   /* 1 MB - Minimum size of each compression job */
 #endif
 
-ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize,  /**< dict can be released after init, a local copy is preserved within zcs */
-                                          ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
+                                        const void* dict, size_t dictSize,   /* dict can be released after init, a local copy is preserved within zcs */
+                                        ZSTD_parameters params,
+                                        unsigned long long pledgedSrcSize);  /* pledgedSrcSize is optional and can be zero == unknown */
 
 
 /* ZSDTMT_parameter :
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 68ec13d1a..bb3eed399 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -70,19 +70,6 @@
 #define FSE_isError  ERR_isError
 #define HUF_isError  ERR_isError
 
-#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
-#  include 
-   static unsigned g_debugLevel = ZSTD_DEBUG;
-#  define DEBUGLOG(l, ...) {                          \
-                if (l<=g_debugLevel) {                \
-                    fprintf(stderr, __FILE__ ": ");   \
-                    fprintf(stderr, __VA_ARGS__);     \
-                    fprintf(stderr, " \n");           \
-            }   }
-#else
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
-#endif
-
 
 /*_*******************************************************
 *  Memory operations

From 8c910d2097899f44ddbe90a2251655fad4c569fa Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 3 Jun 2017 01:15:02 -0700
Subject: [PATCH 054/109] updated ZSTDMT streaming API

ZSTDMT streaming API is now similar
and has same capabilites as single-thread streaming API.
It makes it easier to blend them together.
---
 doc/zstd_manual.html           |  2 +-
 lib/common/bitstream.h         |  1 +
 lib/common/zstd_internal.h     | 29 ++++++++++++
 lib/compress/zstd_compress.c   | 83 ++++++++++++++++++----------------
 lib/compress/zstdmt_compress.c | 80 +++++++++++++++++++-------------
 lib/compress/zstdmt_compress.h |  4 ++
 lib/zstd.h                     |  2 +-
 tests/zstreamtest.c            |  8 ++--
 8 files changed, 132 insertions(+), 77 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 40cae8f28..9e565139c 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -801,7 +801,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
 


 
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
  start a new compression job, using same parameters from previous job.
diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index f3c9064bb..07b85026c 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -75,6 +75,7 @@ extern "C" {
 #define STREAM_ACCUMULATOR_MIN_64  57
 #define STREAM_ACCUMULATOR_MIN    ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
 
+
 /*-******************************************
 *  bitStream encoding API (write forward)
 ********************************************/
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 8c627cb7a..e67157c13 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -305,4 +305,33 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
 void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
 
 
+/*! ZSTD_compressBegin_internal() :
+ *  innermost initialization function. Private use only.
+ *  expects params to be valid.
+ *  must receive dict, or cdict, or none, but not both.
+ *  @return : 0, or an error code */
+typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
+size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
+                             const void* dict, size_t dictSize,
+                             const ZSTD_CDict* cdict,
+                                   ZSTD_parameters params, U64 pledgedSrcSize,
+                                   ZSTD_buffered_policy_e zbuff);
+
+
+/*! ZSTD_initCStream_internal() :
+ *  Private use only. Init streaming operation.
+ *  expects params to be valid.
+ *  must receive dict, or cdict, or none, but not both.
+ *  @return : 0, or an error code */
+size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
+                           const void* dict, size_t dictSize,
+                           const ZSTD_CDict* cdict,
+                           ZSTD_parameters params, U64 pledgedSrcSize);
+
+
+/*! ZSTD_getParamsFromCDict() :
+ *  as the name implies */
+ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict);
+
+
 #endif   /* ZSTD_CCOMMON_H_MODULE */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 76d3614c7..083f891b3 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -74,7 +74,7 @@ struct ZSTD_CDict_s {
     const void* dictContent;
     size_t dictContentSize;
     ZSTD_CCtx* refContext;
-};  /* typedef'd tp ZSTD_CDict within "zstd.h" */
+};  /* typedef'd to ZSTD_CDict within "zstd.h" */
 
 struct ZSTD_CCtx_s {
     const BYTE* nextSrc;    /* next block here to continue on current prefix */
@@ -533,8 +533,6 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
 
 typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e;
 
-typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
-
 /*! ZSTD_resetCCtx_internal() :
     note : `params` are assumed fully validated at this stage */
 static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
@@ -3089,7 +3087,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict,
 
 /*! ZSTD_compressBegin_internal() :
 *   @return : 0, or an error code */
-static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
+size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
                              const ZSTD_CDict* cdict,
                                    ZSTD_parameters params, U64 pledgedSrcSize,
@@ -3392,7 +3390,7 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
     return cdict;
 }
 
-static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
+ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
     return ZSTD_getParamsFromCCtx(cdict->refContext);
 }
 
@@ -3505,38 +3503,12 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
 }
 
-/* ZSTD_initCStream_usingCDict_advanced() :
- * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
-size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
-                                            const ZSTD_CDict* cdict,
-                                            unsigned long long pledgedSrcSize,
-                                            ZSTD_frameParameters fParams)
-{   /* cannot handle NULL cdict (does not know what to do) */
-    if (!cdict) return ERROR(dictionary_wrong);
-    {   ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
-        params.fParams = fParams;
-        zcs->requestedParams = params;
-        zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
-        zcs->cdict = cdict;
-        return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
-    }
-}
-
-/* note : cdict must outlive compression session */
-size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
-{
-    ZSTD_frameParameters const fParams = { 0 /* contentSize */, 0 /* checksum */, 0 /* hideDictID */ };
-    /* cannot handle NULL cdict (does not know what to do) */
-    if (!cdict) return ERROR(dictionary_wrong);
-    return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, 0, fParams);
-}
-
-static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
-                                const void* dict, size_t dictSize,
-                                ZSTD_parameters params, unsigned long long pledgedSrcSize)
+size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
+                    const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
+                    ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
-    zcs->cdict = NULL;
+    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
     if (dict && dictSize >= 8) {
         if (zcs->staticSize) {   /* static CCtx : never uses malloc */
@@ -3544,14 +3516,46 @@ static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
             return ERROR(memory_allocation);
         }
         ZSTD_freeCDict(zcs->cdictLocal);
+        zcs->cdict = NULL;
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params.cParams, zcs->customMem);
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
         zcs->cdict = zcs->cdictLocal;
+    } else {
+        if (cdict) {
+            ZSTD_parameters const cdictParams = ZSTD_getParamsFromCDict(cdict);
+            params.cParams = cdictParams.cParams;  /* cParams are enforced from cdict */
+        }
+        zcs->cdict = cdict;
     }
 
+    zcs->requestedParams = params;
+    zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
     return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
 }
 
+/* ZSTD_initCStream_usingCDict_advanced() :
+ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
+size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
+                                            const ZSTD_CDict* cdict,
+                                            ZSTD_frameParameters fParams,
+                                            unsigned long long pledgedSrcSize)
+{   /* cannot handle NULL cdict (does not know what to do) */
+    if (!cdict) return ERROR(dictionary_wrong);
+    {   ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
+        params.fParams = fParams;
+        return ZSTD_initCStream_internal(zcs,
+                                NULL, 0, cdict,
+                                params, pledgedSrcSize);
+    }
+}
+
+/* note : cdict must outlive compression session */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
+{
+    ZSTD_frameParameters const fParams = { 0 /* contentSize */, 0 /* checksum */, 0 /* hideDictID */ };
+    return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0);  /* note : will check that cdict != NULL */
+}
+
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  const void* dict, size_t dictSize,
                                  ZSTD_parameters params, unsigned long long pledgedSrcSize)
@@ -3559,27 +3563,27 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
     CHECK_F( ZSTD_checkCParams(params.cParams) );
     zcs->requestedParams = params;
     zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
-    return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize);
+    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, params, pledgedSrcSize);
 }
 
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
     zcs->compressionLevel = compressionLevel;
-    return ZSTD_initCStream_internal(zcs, dict, dictSize, params, 0);
+    return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, params, 0);
 }
 
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
 {
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
     params.fParams.contentSizeFlag = (pledgedSrcSize>0);
-    return ZSTD_initCStream_internal(zcs, NULL, 0, params, pledgedSrcSize);
+    return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, params, pledgedSrcSize);
 }
 
 size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0);
-    return ZSTD_initCStream_internal(zcs, NULL, 0, params, 0);
+    return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, params, 0);
 }
 
 /*======   Compression   ======*/
@@ -3606,6 +3610,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
     U32 someMoreWork = 1;
 
     /* check expectations */
+    DEBUGLOG(5, "ZSTD_compressStream_generic");
     assert(zcs->inBuff != NULL);
     assert(zcs->outBuff!= NULL);
     assert(output->pos <= output->size);
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 54feb86f5..c71dcd63b 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -27,19 +27,12 @@
 
 
 /* ======   Debug   ====== */
-#if defined(ZSTDMT_DEBUG) && (ZSTDMT_DEBUG>=2)
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
 
 #  include 
 #  include 
 #  include 
-#  define DEBUGLOGRAW(l, ...) if (l<=ZSTDMT_DEBUG) { fprintf(stderr, __VA_ARGS__); }
-#  define DEBUGLOG(l, ...) {            \
-    if (l<=ZSTDMT_DEBUG) {              \
-        fprintf(stderr, __FILE__ ": "); \
-        fprintf(stderr, __VA_ARGS__);   \
-        fprintf(stderr, " \n");         \
-    }                                   \
-}
+#  define DEBUGLOGRAW(l, ...) if (l<=ZSTD_DEBUG) { fprintf(stderr, __VA_ARGS__); }
 
 #  define DEBUG_PRINTHEX(l,p,n) {            \
     unsigned debug_u;                        \
@@ -59,7 +52,7 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
 
 #define MUTEX_WAIT_TIME_DLEVEL 5
 #define PTHREAD_MUTEX_LOCK(mutex) {               \
-    if (ZSTDMT_DEBUG>=MUTEX_WAIT_TIME_DLEVEL) {   \
+    if (ZSTD_DEBUG>=MUTEX_WAIT_TIME_DLEVEL) {   \
         unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \
         pthread_mutex_lock(mutex);                \
         {   unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \
@@ -73,7 +66,6 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
 
 #else
 
-#  define DEBUGLOG(l, ...)      {}    /* disabled */
 #  define PTHREAD_MUTEX_LOCK(m) pthread_mutex_lock(m)
 #  define DEBUG_PRINTHEX(l,p,n) {}
 
@@ -259,7 +251,7 @@ typedef struct {
     pthread_mutex_t* jobCompleted_mutex;
     pthread_cond_t* jobCompleted_cond;
     ZSTD_parameters params;
-    ZSTD_CDict* cdict;
+    const ZSTD_CDict* cdict;
     unsigned long long fullFrameSize;
 } ZSTDMT_jobDescription;
 
@@ -273,7 +265,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
                  job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
     if (job->cdict) {  /* should only happen for first segment */
         size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize);
-        if (job->cdict) DEBUGLOG(3, "using CDict");
+        DEBUGLOG(3, "using CDict");
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
     } else {  /* srcStart points at reloaded section */
         if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0;  /* ensure no srcSize control */
@@ -335,7 +327,8 @@ struct ZSTDMT_CCtx_s {
     unsigned long long frameContentSize;
     size_t sectionSize;
     ZSTD_customMem cMem;
-    ZSTD_CDict* cdict;
+    ZSTD_CDict* cdictLocal;
+    const ZSTD_CDict* cdict;
 };
 
 ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
@@ -407,7 +400,7 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
     ZSTDMT_freeBufferPool(mtctx->buffPool);  /* release job resources into pools first */
     ZSTD_free(mtctx->jobs, mtctx->cMem);
     ZSTDMT_freeCCtxPool(mtctx->cctxPool);
-    ZSTD_freeCDict(mtctx->cdict);
+    ZSTD_freeCDict(mtctx->cdictLocal);
     pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
     pthread_cond_destroy(&mtctx->jobCompleted_cond);
     ZSTD_free(mtctx, mtctx->cMem);
@@ -422,7 +415,7 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
         + ZSTDMT_sizeof_bufferPool(mtctx->buffPool)
         + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription)
         + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool)
-        + ZSTD_sizeof_CDict(mtctx->cdict);
+        + ZSTD_sizeof_CDict(mtctx->cdictLocal);
 }
 
 size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value)
@@ -567,28 +560,38 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
 
 
 static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
-                        const void* dict, size_t dictSize, unsigned updateDict,
+                        const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
                         ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
-    if (zcs->nbThreads==1)
-        return ZSTD_initCStream_advanced(zcs->cctxPool->cctx[0],
-                                        dict, dictSize,
-                                        params, pledgedSrcSize);
+    /* params are supposed to be fully validated at this point */
+    assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
+    assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
+
+    if (zcs->nbThreads==1) {
+        return ZSTD_initCStream_internal(zcs->cctxPool->cctx[0],
+                                dict, dictSize, cdict,
+                                params, pledgedSrcSize);
+    }
 
     if (zcs->allJobsCompleted == 0) {   /* previous compression not correctly finished */
         ZSTDMT_waitForAllJobsCompleted(zcs);
         ZSTDMT_releaseAllJobResources(zcs);
         zcs->allJobsCompleted = 1;
     }
+
     zcs->params = params;
-    if (updateDict) {
-        ZSTD_freeCDict(zcs->cdict); zcs->cdict = NULL;
-        if (dict && dictSize) {
-            zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, 0 /* byRef */,
-                                                params.cParams, zcs->cMem);
-            if (zcs->cdict == NULL) return ERROR(memory_allocation);
-    }   }
     zcs->frameContentSize = pledgedSrcSize;
+    if (dict) {
+        ZSTD_freeCDict(zcs->cdictLocal);
+        zcs->cdict = NULL;
+        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* byRef */,
+                                                params.cParams, zcs->cMem);
+        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
+        zcs->cdict = zcs->cdictLocal;
+    } else {
+        zcs->cdict = cdict;
+    }
+
     zcs->targetDictSize = (zcs->overlapRLog>=9) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog);
     DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog);
     DEBUGLOG(3, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
@@ -610,13 +613,25 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     return 0;
 }
 
-size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* zcs,
+size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
                                 const void* dict, size_t dictSize,
                                 ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
-    return ZSTDMT_initCStream_internal(zcs, dict, dictSize, 1, params, pledgedSrcSize);
+    return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, NULL, params, pledgedSrcSize);
 }
 
+size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
+                               const ZSTD_CDict* cdict,
+                                     ZSTD_frameParameters fParams,
+                                     unsigned long long pledgedSrcSize)
+{
+    ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
+    if (cdict==NULL) return ERROR(GENERIC);   /* method incompatible with NULL cdict */
+    params.fParams = fParams;
+    return ZSTDMT_initCStream_internal(mtctx, NULL, 0, cdict, params, pledgedSrcSize);
+}
+
+
 /* ZSTDMT_resetCStream() :
  * pledgedSrcSize is optional and can be zero == unknown */
 size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
@@ -628,7 +643,7 @@ size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize)
 
 size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0);
-    return ZSTDMT_initCStream_internal(zcs, NULL, 0, 1, params, 0);
+    return ZSTDMT_initCStream_internal(zcs, NULL, 0, NULL, params, 0);
 }
 
 
@@ -780,8 +795,9 @@ size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBu
     if (zcs->frameEnded)
         /* current frame being ended. Only flush is allowed. Restart with init */
         return ERROR(stage_wrong);
-    if (zcs->nbThreads==1)
+    if (zcs->nbThreads==1) {
         return ZSTD_compressStream(zcs->cctxPool->cctx[0], output, input);
+    }
 
     /* fill input buffer */
     {   size_t const toLoad = MIN(input->size - input->pos, zcs->inBuffSize - zcs->inBuff.filled);
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index ceff32ef6..267ed3e2f 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -65,6 +65,10 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
                                         ZSTD_parameters params,
                                         unsigned long long pledgedSrcSize);  /* pledgedSrcSize is optional and can be zero == unknown */
 
+ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
+                                        const ZSTD_CDict* cdict,
+                                        ZSTD_frameParameters fparams,
+                                        unsigned long long pledgedSrcSize);  /* note : zero means empty */
 
 /* ZSDTMT_parameter :
  * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
diff --git a/lib/zstd.h b/lib/zstd.h
index c71ed493e..b7fbbea1c 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -932,7 +932,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
+ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize);  /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
 
 /*! ZSTD_resetCStream() :
  *  start a new compression job, using same parameters from previous job.
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 0f73b8cd4..17772856e 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -486,7 +486,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
         ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
         ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1 /* byReference */, cParams, customMem);
-        size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, CNBufferSize, fParams);
+        size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
         if (ZSTD_isError(initError)) goto _output_error;
         cSize = 0;
         outBuff.dst = compressedBuffer;
@@ -497,7 +497,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
         inBuff.pos = 0;
         { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
           if (ZSTD_isError(r)) goto _output_error; }
-        if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
+        if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
         { size_t const r = ZSTD_endStream(zc, &outBuff);
           if (r != 0) goto _output_error; }  /* error, or some data not flushed */
         cSize = outBuff.pos;
@@ -991,8 +991,8 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
             U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
             U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
             U32 const cLevel = (FUZ_rand(&lseed) %
-                                (ZSTD_maxCLevel() -
-                                 (MAX(testLog, dictLog) / cLevelLimiter))) +
+                               (ZSTD_maxCLevel() -
+                               (MAX(testLog, dictLog) / cLevelLimiter))) +
                                1;
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
             oldTestLog = testLog;

From 51235393e30beea59aa6c17908c71a4f7d0da818 Mon Sep 17 00:00:00 2001
From: cyan4973 
Date: Sun, 4 Jun 2017 22:06:25 -0700
Subject: [PATCH 055/109] fixed fullbench project for VS2010+

---
 build/VS2010/fullbench/fullbench.vcxproj | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj
index e16f5e1db..2bff4ca47 100644
--- a/build/VS2010/fullbench/fullbench.vcxproj
+++ b/build/VS2010/fullbench/fullbench.vcxproj
@@ -156,26 +156,32 @@
   
   
     
+    
     
     
+    
+    
     
-    
     
     
     
+    
     
     
     
     
   
   
+    
     
     
-    
     
-    
     
+    
+    
+    
     
+    
     
     
     

From 8bcbf42617c7cdda17911614d54e61b6e41cfc34 Mon Sep 17 00:00:00 2001
From: cyan4973 
Date: Sun, 4 Jun 2017 23:52:00 -0700
Subject: [PATCH 056/109] fixed g++ prototype mismatch

---
 Makefile                    |   2 +-
 lib/common/zstd_internal.h  |   6 +++---
 programs/windres/zstd32.res | Bin 1044 -> 1044 bytes
 programs/windres/zstd64.res | Bin 1044 -> 1044 bytes
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 54652665b..d7a00e431 100644
--- a/Makefile
+++ b/Makefile
@@ -108,7 +108,7 @@ clean:
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, OSX, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD MSYS_NT))
 
 HOST_OS = POSIX
 CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index e67157c13..bd2ad10ff 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -324,9 +324,9 @@ size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
  *  must receive dict, or cdict, or none, but not both.
  *  @return : 0, or an error code */
 size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
-                           const void* dict, size_t dictSize,
-                           const ZSTD_CDict* cdict,
-                           ZSTD_parameters params, U64 pledgedSrcSize);
+                     const void* dict, size_t dictSize,
+                     const ZSTD_CDict* cdict,
+                     ZSTD_parameters params, unsigned long long pledgedSrcSize);
 
 
 /*! ZSTD_getParamsFromCDict() :
diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res
index b5dd78db717ce9c7ddb2f69a9bc81804e51aedf6..d6caf985ed2a9469d7a06c8518fc8af5780e34d5 100644
GIT binary patch
delta 47
qcmbQjF@
Date: Mon, 5 Jun 2017 00:12:13 -0700
Subject: [PATCH 057/109] minor fix for -Wdocumentation

---
 lib/common/zstd_internal.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index bd2ad10ff..77f6e5672 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -305,12 +305,12 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
 void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
 
 
+typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
 /*! ZSTD_compressBegin_internal() :
  *  innermost initialization function. Private use only.
  *  expects params to be valid.
  *  must receive dict, or cdict, or none, but not both.
  *  @return : 0, or an error code */
-typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
 size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
                              const ZSTD_CDict* cdict,

From f35e2de61c32fadc4ab3876c4c8029556bfa0bd1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 5 Jun 2017 18:32:48 -0700
Subject: [PATCH 058/109] linked newAPI to ZSTDMT

---
 doc/zstd_manual.html           |  2 +-
 lib/compress/zstd_compress.c   | 22 +++++++++++++++++++---
 lib/compress/zstdmt_compress.c | 25 ++++++++++++++++++++++---
 lib/compress/zstdmt_compress.h | 13 +++++++++++++
 lib/zstd.h                     |  2 +-
 5 files changed, 56 insertions(+), 8 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 9e565139c..523c49bb4 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -707,7 +707,7 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
 
  Same as ZSTD_compress_generic(),
   but using only simple integral types as arguments.
   Argument list is less expressive than ZSTD_{in,out}Buffer,
-  but can be helpful for binders towards dynamic languages
+  but can be helpful for binders to dynamic languages
   which have troubles handling structures containing memory pointers.
  
 


diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 083f891b3..48eb24ddf 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3734,6 +3734,16 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf
     return ZSTD_compressStream_generic(zcs, output, input, ZSTD_e_continue);
 }
 
+/*! ZSTDMT_initCStream_internal() :
+ *  Private use only. Init streaming operation.
+ *  expects params to be valid.
+ *  must receive dict, or cdict, or none, but not both.
+ *  @return : 0, or an error code */
+size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
+                    const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
+                    ZSTD_parameters params, unsigned long long pledgedSrcSize);
+
+
 size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
                               ZSTD_outBuffer* output,
                               ZSTD_inBuffer* input,
@@ -3750,12 +3760,19 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
                                     cctx->frameContentSize, 0 /* dictSize */);
-        CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->frameContentSize) );
+        if (cctx->nbThreads > 1) {
+            CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, NULL, 0, cctx->cdict, params, cctx->frameContentSize) );
+        } else {
+            CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->frameContentSize) );
+    }   }
+
+    if (cctx->nbThreads > 1) {
+        DEBUGLOG(5, "starting ZSTDMT_compressStream_generic");
+        return ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp) ;
     }
 
     DEBUGLOG(5, "starting ZSTD_compressStream_generic");
     CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
-
     DEBUGLOG(5, "completing ZSTD_compress_generic");
     return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
 }
@@ -3773,7 +3790,6 @@ size_t ZSTD_compress_generic_simpleArgs (
     *dstPos = output.pos;
     *srcPos = input.pos;
     return cErr;
-
 }
 
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index c71dcd63b..09ac51218 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -559,9 +559,9 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
 }
 
 
-static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
-                        const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
-                        ZSTD_parameters params, unsigned long long pledgedSrcSize)
+size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
+                    const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
+                    ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     /* params are supposed to be fully validated at this point */
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
@@ -849,3 +849,22 @@ size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
         return ZSTD_endStream(zcs->cctxPool->cctx[0], output);
     return ZSTDMT_flushStream_internal(zcs, output, 1);
 }
+
+size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
+                                    ZSTD_outBuffer* output,
+                                    ZSTD_inBuffer* input,
+                                    ZSTD_EndDirective endOp)
+{
+    CHECK_F (ZSTDMT_compressStream(mtctx, output, input));
+    switch(endOp)
+    {
+        case ZSTD_e_flush:
+            return ZSTDMT_flushStream(mtctx, output);
+        case ZSTD_e_end:
+            return ZSTDMT_endStream(mtctx, output);
+        case ZSTD_e_continue:
+            return 1;
+        default:
+            return ERROR(GENERIC);
+    }
+}
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 267ed3e2f..8b0ca824f 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -85,6 +85,19 @@ typedef enum {
 ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, unsigned value);
 
 
+/*! ZSTDMT_compressStream_generic() :
+ *  Combines ZSTDMT_compressStream() with ZSTDMT_flushStream() or ZSTDMT_endStream()
+ *  depending on flush directive
+ * @return : minimum amount of data still to be flushed
+ *           0 if fully flushed
+ *           or an error code */
+ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
+                                                ZSTD_outBuffer* output,
+                                                ZSTD_inBuffer* input,
+                                                ZSTD_EndDirective endOp);
+
+
+
 #if defined (__cplusplus)
 }
 #endif
diff --git a/lib/zstd.h b/lib/zstd.h
index b7fbbea1c..357af9194 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -828,7 +828,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
  *  Same as ZSTD_compress_generic(),
  *  but using only simple integral types as arguments.
  *  Argument list is less expressive than ZSTD_{in,out}Buffer,
- *  but can be helpful for binders towards dynamic languages
+ *  but can be helpful for binders to dynamic languages
  *  which have troubles handling structures containing memory pointers.
  */
 size_t ZSTD_compress_generic_simpleArgs (

From 23aace9778745c9f3bf6246f3d92abc8c9b8193b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 11 Jun 2017 18:32:36 -0700
Subject: [PATCH 059/109] added control stage to MT mode

---
 lib/compress/zstd_compress.c   | 6 ++++--
 lib/compress/zstdmt_compress.c | 8 ++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 48eb24ddf..338dd7472 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3762,13 +3762,15 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
                                     cctx->frameContentSize, 0 /* dictSize */);
         if (cctx->nbThreads > 1) {
             CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, NULL, 0, cctx->cdict, params, cctx->frameContentSize) );
+            cctx->streamStage = zcss_load;
         } else {
             CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->frameContentSize) );
     }   }
 
     if (cctx->nbThreads > 1) {
-        DEBUGLOG(5, "starting ZSTDMT_compressStream_generic");
-        return ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp) ;
+        size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
+        if (ZSTD_isError(flushMin)) cctx->streamStage = zcss_init;
+        return flushMin;
     }
 
     DEBUGLOG(5, "starting ZSTD_compressStream_generic");
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 09ac51218..098412824 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -851,11 +851,11 @@ size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
 }
 
 size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
-                                    ZSTD_outBuffer* output,
-                                    ZSTD_inBuffer* input,
-                                    ZSTD_EndDirective endOp)
+                                     ZSTD_outBuffer* output,
+                                     ZSTD_inBuffer* input,
+                                     ZSTD_EndDirective endOp)
 {
-    CHECK_F (ZSTDMT_compressStream(mtctx, output, input));
+    CHECK_F(ZSTDMT_compressStream(mtctx, output, input));
     switch(endOp)
     {
         case ZSTD_e_flush:

From 9e6a2eaab6941bad2f34708b0057d4314dee1988 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 11 Jun 2017 18:39:46 -0700
Subject: [PATCH 060/109] added MT support to NEWAPI

---
 programs/fileio.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index e7c58a969..bbf52168a 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -379,10 +379,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
         size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName);   /* works with dictFileName==NULL */
         if (dictFileName && (dictBuffer==NULL))
             EXM_THROW(32, "allocation error : can't create dictBuffer");
-//#define ZSTD_NEWAPI
+
 #ifdef ZSTD_NEWAPI
-        {
-            /* frame parameters */
+        {   /* frame parameters */
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, srcIsRegularFile) );
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
@@ -396,6 +395,8 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
             CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
+            /* multi-threading */
+            CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
             /* dictionary */
             CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
         }

From f129fd39703183920ecd673bf6a94a2322b92771 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 11 Jun 2017 18:46:09 -0700
Subject: [PATCH 061/109] disabled MT code path when ZSTD_MULTITHREAD is not
 defined

---
 lib/compress/zstd_compress.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 338dd7472..db672da04 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3760,18 +3760,24 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
                                     cctx->frameContentSize, 0 /* dictSize */);
+
+#ifdef ZSTD_MULTITHREAD
         if (cctx->nbThreads > 1) {
             CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, NULL, 0, cctx->cdict, params, cctx->frameContentSize) );
             cctx->streamStage = zcss_load;
-        } else {
+        } else
+#endif
+        {
             CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->frameContentSize) );
     }   }
 
+#ifdef ZSTD_MULTITHREAD
     if (cctx->nbThreads > 1) {
         size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
         if (ZSTD_isError(flushMin)) cctx->streamStage = zcss_init;
         return flushMin;
     }
+#endif
 
     DEBUGLOG(5, "starting ZSTD_compressStream_generic");
     CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );

From bb0aaf9579a295594bc92b2d0762829c0c609508 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 12 Jun 2017 05:19:15 -0700
Subject: [PATCH 062/109] minor man update on -B# option in benchmark mode

---
 programs/zstd.1    | 6 +++---
 programs/zstd.1.md | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/programs/zstd.1 b/programs/zstd.1
index 427146795..7e2e99159 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "May 2017" "zstd 1.3.0" "User Commands"
+.TH "ZSTD" "1" "June 2017" "zstd 1.3.0" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -235,8 +235,8 @@ benchmark file(s) using multiple compression levels, from \fB\-b#\fR to \fB\-e#\
 minimum evaluation time, in seconds (default : 3s), benchmark mode only
 .
 .TP
-\fB\-B#\fR
-cut file into independent blocks of size # (default: no block)
+\fB\-B#\fR, \fB\-\-block\-size=#\fR
+cut file(s) into independent blocks of size # (default: no block)
 .
 .TP
 \fB\-\-priority=rt\fR
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 2f5a6747e..4e731ddba 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -230,8 +230,8 @@ BENCHMARK
     benchmark file(s) using multiple compression levels, from `-b#` to `-e#` (inclusive)
 * `-i#`:
     minimum evaluation time, in seconds (default : 3s), benchmark mode only
-* `-B#`:
-    cut file into independent blocks of size # (default: no block)
+* `-B#`, `--block-size=#`:
+    cut file(s) into independent blocks of size # (default: no block)
 * `--priority=rt`:
     set process priority to real-time
 

From 05ae4b2190e9e11cd850fdc599f1224a00802eb6 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 15 Jun 2017 18:03:34 -0700
Subject: [PATCH 063/109] added protection : MT incompatible with Static
 allocation

---
 lib/compress/zstd_compress.c | 2 ++
 lib/zstd.h                   | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index db672da04..8377b37e5 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -339,6 +339,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         if (value > 1) return ERROR(compressionParameter_unsupported);
 #endif
         if ((value>1) && (cctx->nbThreads != value)) {
+            if (cctx->staticSize)  /* MT not compatible with static alloc */
+                return ERROR(compressionParameter_unsupported);
             ZSTDMT_freeCCtx(cctx->mtctx);
             cctx->nbThreads = value;
             cctx->mtctx = ZSTDMT_createCCtx(value);
diff --git a/lib/zstd.h b/lib/zstd.h
index 357af9194..eb72fc4e5 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -68,7 +68,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< to be used when checking d
 #define ZSTD_QUOTE(str) #str
 #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
 #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
-ZSTDLIB_API const char* ZSTD_versionString(void);
+ZSTDLIB_API const char* ZSTD_versionString(void);   /* v1.3.0 */
 
 
 /***************************************
@@ -424,6 +424,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 /* use this constant to defer to stdlib's functions */
 static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL};
 
+
 /***************************************
 *  Frame size functions
 ***************************************/

From cc9f9b7f4c3c5e00f78b6c7ca9498297ece121ce Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 15 Jun 2017 18:17:10 -0700
Subject: [PATCH 064/109] protection : ZSTD_CONTENTSIZE_UNKNOWN automatically
 disables contentSizeFlag

---
 lib/compress/zstd_compress.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 8377b37e5..1d4f4f5a1 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -542,8 +542,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                                       ZSTD_compResetPolicy_e const crp,
                                       ZSTD_buffered_policy_e const zbuff)
 {
-    DEBUGLOG(5, "ZSTD_resetCCtx_internal : wlog=%u / old=%u",
-                params.cParams.windowLog, zc->appliedParams.cParams.windowLog);
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
 
     if (crp == ZSTDcrp_continue) {
@@ -612,10 +610,11 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
         /* init params */
         zc->appliedParams = params;
-        zc->blockSize = blockSize;
-        DEBUGLOG(5, "blockSize = %uK", (U32)blockSize>>10);
         zc->frameContentSize = frameContentSize;
         zc->consumedSrcSize = 0;
+        if (frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN)
+            zc->appliedParams.fParams.contentSizeFlag = 0;
+        zc->blockSize = blockSize;
 
         XXH64_reset(&zc->xxhState, 0);
         zc->stage = ZSTDcs_init;

From bd18c885a38d7cd93fe610c7e647e280198ada50 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 10:17:50 -0700
Subject: [PATCH 065/109] added ZSTD_CCtx_reset

---
 lib/compress/zstd_compress.c | 11 +++++++++--
 lib/zstd.h                   |  5 +++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1d4f4f5a1..1be0e97b3 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -399,14 +399,14 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
 }
 
 /* Not ready yet ! */
-ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
+size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
 {
     (void)cctx; (void)prefix; (void)prefixSize; /* to be done later */
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     return ERROR(compressionParameter_unsupported);
 }
 
-ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
+size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     cctx->cdict = cdict;
@@ -414,6 +414,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 }
 
 
+void ZSTD_CCtx_reset(ZSTD_CCtx* cctx)
+{
+    cctx->streamStage = zcss_init;
+    cctx->frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
+    cctx->cdict = NULL;
+}
+
 /** ZSTD_checkParams() :
     ensure param values remain within authorized range.
     @return : 0, or an error code if one value is beyond authorized range */
diff --git a/lib/zstd.h b/lib/zstd.h
index eb72fc4e5..a19543814 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -819,10 +819,11 @@ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
 /*! ZSTD_CCtx_reset() :
  *  Return a CCtx to clean state.
  *  Useful after an error, or to interrupt an ongoing compression job and start a new one.
- *  It's possible to modify compression parameters after a reset.
  *  Any internal data not yet flushed is cancelled.
+ *  Dictionary (if any) is dropped, next compression starts with srcSize==unknown by default.
+ *  It's possible to modify compression parameters after a reset.
  */
-ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
+ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
 
 
 /*! ZSTD_compress_generic_simpleArgs() :

From 559ee82e901020f7fe98b1beec402298a57f8623 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 11:58:21 -0700
Subject: [PATCH 066/109] fixed : calling ZSTD_compress_generic() to end-flush
 a stream in multiple steps

---
 lib/common/error_private.c     |  3 ++-
 lib/common/zstd_errors.h       | 10 +++++++---
 lib/compress/zstd_compress.c   | 32 ++++++++++++++++++--------------
 lib/compress/zstdmt_compress.c | 25 +++++++++++++++----------
 lib/zstd.h                     |  4 ++--
 5 files changed, 44 insertions(+), 30 deletions(-)

diff --git a/lib/common/error_private.c b/lib/common/error_private.c
index 6bc86da7a..2d752cd23 100644
--- a/lib/common/error_private.c
+++ b/lib/common/error_private.c
@@ -24,7 +24,8 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
     case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
     case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
-    case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
+    case PREFIX(compressionParameter_unsupported): return "Compression parameter is not supported";
+    case PREFIX(compressionParameter_outOfBound): return "Compression parameter is out of bound";
     case PREFIX(init_missing): return "Context should be init first";
     case PREFIX(memory_allocation): return "Allocation error : not enough memory";
     case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h
index e067acf6e..19f1597aa 100644
--- a/lib/common/zstd_errors.h
+++ b/lib/common/zstd_errors.h
@@ -35,8 +35,11 @@ extern "C" {
 #endif
 
 /*-****************************************
-*  error codes list
-******************************************/
+ *  error codes list
+ *  note : this API is still considered unstable
+ *         it should not be used with a dynamic library
+ *         only static linking is allowed
+ ******************************************/
 typedef enum {
   ZSTD_error_no_error,
   ZSTD_error_GENERIC,
@@ -47,6 +50,7 @@ typedef enum {
   ZSTD_error_frameParameter_unsupportedBy32bits,
   ZSTD_error_frameParameter_windowTooLarge,
   ZSTD_error_compressionParameter_unsupported,
+  ZSTD_error_compressionParameter_outOfBound,
   ZSTD_error_init_missing,
   ZSTD_error_memory_allocation,
   ZSTD_error_stage_wrong,
@@ -67,7 +71,7 @@ typedef enum {
 
 /*! ZSTD_getErrorCode() :
     convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
-    which can be used to compare directly with enum list published into "error_public.h" */
+    which can be used to compare with enum list published above */
 ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
 ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1be0e97b3..dfaa1854f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -241,7 +241,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 {
 #   define CLAMPCHECK(val,min,max) {                         \
         if ((valmax)) {                         \
-            return ERROR(compressionParameter_unsupported);  \
+            return ERROR(compressionParameter_outOfBound);  \
     }   }
 
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
@@ -342,19 +342,20 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
             if (cctx->staticSize)  /* MT not compatible with static alloc */
                 return ERROR(compressionParameter_unsupported);
             ZSTDMT_freeCCtx(cctx->mtctx);
-            cctx->nbThreads = value;
+            cctx->nbThreads = 1;
             cctx->mtctx = ZSTDMT_createCCtx(value);
             if (cctx->mtctx == NULL) return ERROR(memory_allocation);
-        }
-        cctx->nbThreads = 1;
+            cctx->nbThreads = value;
+        } else
+            cctx->nbThreads = 1;
         return 0;
 
-    case ZSTDMT_p_jobSize:
+    case ZSTD_p_jobSize:
         if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
         assert(cctx->mtctx != NULL);
         return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_sectionSize, value);
 
-    case ZSTDMT_p_overlapSizeLog:
+    case ZSTD_p_overlapSizeLog:
         if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
         assert(cctx->mtctx != NULL);
         return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
@@ -3648,8 +3649,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                   && (zcs->inBuffPos == zcs->inToCompress) ) {
                     /* empty */
                     someMoreWork = 0; break;
-                 }
-             }
+                }
+            }
             /* compress current block (note : this stage cannot be stopped in the middle) */
             DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
             {   void* cDst;
@@ -3658,7 +3659,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 size_t oSize = oend-op;
                 unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
                 if (oSize >= ZSTD_compressBound(iSize))
-                    cDst = op;   /* compress directly into output buffer (skip flush stage) */
+                    cDst = op;   /* compress into output buffer, to skip flush stage */
                 else
                     cDst = zcs->outBuff, oSize = zcs->outBuffSize;
                 cSize = lastBlock ?
@@ -3667,7 +3668,6 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                         ZSTD_compressContinue(zcs, cDst, oSize,
                                     zcs->inBuff + zcs->inToCompress, iSize);
                 if (ZSTD_isError(cSize)) return cSize;
-                DEBUGLOG(5, "cSize = %u   (lastBlock:%u)", (U32)cSize, lastBlock);
                 zcs->frameEnded = lastBlock;
                 /* prepare next block */
                 zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
@@ -3681,7 +3681,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 if (cDst == op) {  /* no need to flush */
                     op += cSize;
                     if (zcs->frameEnded) {
-                        DEBUGLOG(5, "Frame directly completed");
+                        DEBUGLOG(5, "Frame completed directly in outBuffer");
                         someMoreWork = 0;
                         zcs->streamStage = zcss_init;
                     }
@@ -3707,7 +3707,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 }
                 zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
                 if (zcs->frameEnded) {
-                    DEBUGLOG(5, "Frame completed");
+                    DEBUGLOG(5, "Frame completed on flush");
                     someMoreWork = 0;
                     zcs->streamStage = zcss_init;
                     break;
@@ -3781,15 +3781,19 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
 
 #ifdef ZSTD_MULTITHREAD
     if (cctx->nbThreads > 1) {
+        DEBUGLOG(5, "calling ZSTDMT_compressStream_generic(%i,...)", endOp);
         size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
+        DEBUGLOG(5, "ZSTDMT result : %u", (U32)flushMin);
         if (ZSTD_isError(flushMin)) cctx->streamStage = zcss_init;
+        if (endOp == ZSTD_e_end && flushMin==0)
+            cctx->streamStage = zcss_init;   /* compression completed */
         return flushMin;
     }
 #endif
 
-    DEBUGLOG(5, "starting ZSTD_compressStream_generic");
+    DEBUGLOG(5, "calling ZSTD_compressStream_generic(%i,...)", endOp);
     CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) );
-    DEBUGLOG(5, "completing ZSTD_compress_generic");
+    DEBUGLOG(5, "completed ZSTD_compress_generic");
     return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
 }
 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 098412824..a4b4f5160 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -704,7 +704,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
             zcs->inBuff.filled);
         DEBUGLOG(5, "new inBuff pre-filled");
         zcs->dictSize = newDictSize;
-    } else {
+    } else {   /* if (endFrame==1) */
         zcs->inBuff.buffer = g_nullBuffer;
         zcs->inBuff.filled = 0;
         zcs->dictSize = 0;
@@ -768,7 +768,7 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
             zcs->jobs[wJobID].jobScanned = 1;
         }
         {   size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos);
-            DEBUGLOG(4, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
+            DEBUGLOG(5, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID);
             memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite);
             output->pos += toWrite;
             job.dstFlushed += toWrite;
@@ -808,11 +808,11 @@ size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBu
 
     if ( (zcs->inBuff.filled >= newJobThreshold)  /* filled enough : let's compress */
         && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {   /* avoid overwriting job round buffer */
-        CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0) );
+        CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0 /* blockToFlush */) );
     }
 
     /* check for data to flush */
-    CHECK_F( ZSTDMT_flushNextJob(zcs, output, (zcs->inBuff.filled == zcs->inBuffSize)) ); /* block if it wasn't possible to create new job due to saturation */
+    CHECK_F( ZSTDMT_flushNextJob(zcs, output, (zcs->inBuff.filled == zcs->inBuffSize) /* blockToFlush */) ); /* block if it wasn't possible to create new job due to saturation */
 
     /* recommended next input size : fill current input buffer */
     return zcs->inBuffSize - zcs->inBuff.filled;   /* note : could be zero when input buffer is fully filled and no more availability to create new job */
@@ -823,16 +823,20 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* outp
 {
     size_t const srcSize = zcs->inBuff.filled - zcs->dictSize;
 
-    if (srcSize) DEBUGLOG(4, "flushing : %u bytes left to compress", (U32)srcSize);
+    if (srcSize)
+        DEBUGLOG(5, "flushing : %u bytes left to compress", (U32)srcSize);
     if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded))
        && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {
+        DEBUGLOG(5, "create new job with %u bytes to compress", (U32)srcSize);
+        DEBUGLOG(5, "end order : %u", endFrame);
         CHECK_F( ZSTDMT_createCompressionJob(zcs, srcSize, endFrame) );
+        DEBUGLOG(5, "resulting zcs->frameEnded : %u", zcs->frameEnded);
     }
 
     /* check if there is any data available to flush */
     DEBUGLOG(5, "zcs->doneJobID : %u  ; zcs->nextJobID : %u",
-            zcs->doneJobID, zcs->nextJobID);
-    return ZSTDMT_flushNextJob(zcs, output, 1);
+                zcs->doneJobID, zcs->nextJobID);
+    return ZSTDMT_flushNextJob(zcs, output, 1 /*blockToFlush */);
 }
 
 
@@ -840,14 +844,14 @@ size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
 {
     if (zcs->nbThreads==1)
         return ZSTD_flushStream(zcs->cctxPool->cctx[0], output);
-    return ZSTDMT_flushStream_internal(zcs, output, 0);
+    return ZSTDMT_flushStream_internal(zcs, output, 0 /* endFrame */);
 }
 
 size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
 {
     if (zcs->nbThreads==1)
         return ZSTD_endStream(zcs->cctxPool->cctx[0], output);
-    return ZSTDMT_flushStream_internal(zcs, output, 1);
+    return ZSTDMT_flushStream_internal(zcs, output, 1 /* endFrame */);
 }
 
 size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
@@ -855,7 +859,8 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
                                      ZSTD_inBuffer* input,
                                      ZSTD_EndDirective endOp)
 {
-    CHECK_F(ZSTDMT_compressStream(mtctx, output, input));
+    if (input->pos < input->size)  /* exclude final flushes */
+        CHECK_F(ZSTDMT_compressStream(mtctx, output, input));
     switch(endOp)
     {
         case ZSTD_e_flush:
diff --git a/lib/zstd.h b/lib/zstd.h
index a19543814..1e73b5277 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -702,11 +702,11 @@ typedef enum {
                               * More threads improve speed, but also increase memory usage.
                               * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
                               * Special: value 0 means "do not change nbThreads" */
-    ZSTDMT_p_jobSize,        /* Size of a compression job. Each compression job is completed in parallel.
+    ZSTD_p_jobSize,          /* Size of a compression job. Each compression job is completed in parallel.
                               * 0 means default, which is dynamically determined based on compression parameters.
                               * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
                               * The minimum size is automatically and transparently enforced */
-    ZSTDMT_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
+    ZSTD_p_overlapSizeLog,   /* Size of previous input reloaded at the beginning of each job.
                               * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
 
     /* advanced parameters - may not remain available after API update */

From 33873f0e743376b0554b1e58526e37d2d63c09b6 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 12:04:21 -0700
Subject: [PATCH 067/109] fixed : new advanced AIP : setting nbThreads to the
 same value > 1

---
 lib/compress/zstd_compress.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index dfaa1854f..a482a7f7f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -335,6 +335,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
     case ZSTD_p_nbThreads:
         if (value==0) return 0;
+        DEBUGLOG(5, " setting nbThreads : %u", value);
 #ifndef ZSTD_MULTITHREAD
         if (value > 1) return ERROR(compressionParameter_unsupported);
 #endif
@@ -345,9 +346,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
             cctx->nbThreads = 1;
             cctx->mtctx = ZSTDMT_createCCtx(value);
             if (cctx->mtctx == NULL) return ERROR(memory_allocation);
-            cctx->nbThreads = value;
-        } else
-            cctx->nbThreads = 1;
+        }
+        cctx->nbThreads = value;
         return 0;
 
     case ZSTD_p_jobSize:
@@ -356,6 +356,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_sectionSize, value);
 
     case ZSTD_p_overlapSizeLog:
+        DEBUGLOG(5, " setting overlap with nbThreads == %u", cctx->nbThreads);
         if (cctx->nbThreads <= 1) return ERROR(compressionParameter_unsupported);
         assert(cctx->mtctx != NULL);
         return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);

From 9e73f2f3203482fac5f9ed163513d6205818926f Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 12:24:01 -0700
Subject: [PATCH 068/109] fix : correctly reset pledgedSrcSize to unknown
 status

when starting a new compression with an existing context
---
 lib/compress/zstd_compress.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index a482a7f7f..5334f04f5 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -309,6 +309,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 #endif
 
     case ZSTD_p_contentSizeFlag :
+        DEBUGLOG(2, "set content size flag = %u", (value>0));
         /* Content size written in frame header _when known_ (default:1) */
         cctx->requestedParams.fParams.contentSizeFlag = value>0;
         return 0;
@@ -371,6 +372,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
+    DEBUGLOG(2, " setting pledgedSrcSize to %u", (U32)pledgedSrcSize);
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     cctx->frameContentSize = pledgedSrcSize;
     return 0;
@@ -3198,9 +3200,12 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
     endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
     if (ZSTD_isError(endResult)) return endResult;
     if (cctx->appliedParams.fParams.contentSizeFlag) {  /* control src size */
-        if (cctx->frameContentSize != cctx->consumedSrcSize)
+        DEBUGLOG(2, "end of frame : controlling src size");
+        if (cctx->frameContentSize != cctx->consumedSrcSize) {
+            DEBUGLOG(2, "error : pledgedSrcSize = %u, while realSrcSize = %u",
+                (U32)cctx->frameContentSize, (U32)cctx->consumedSrcSize);
             return ERROR(srcSize_wrong);
-    }
+    }   }
     return cSize + endResult;
 }
 
@@ -3685,6 +3690,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                         DEBUGLOG(5, "Frame completed directly in outBuffer");
                         someMoreWork = 0;
                         zcs->streamStage = zcss_init;
+                        zcs->frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
                     }
                     break;
                 }
@@ -3711,6 +3717,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                     DEBUGLOG(5, "Frame completed on flush");
                     someMoreWork = 0;
                     zcs->streamStage = zcss_init;
+                    zcs->frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
                     break;
                 }
                 zcs->streamStage = zcss_load;
@@ -3786,8 +3793,10 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
         DEBUGLOG(5, "ZSTDMT result : %u", (U32)flushMin);
         if (ZSTD_isError(flushMin)) cctx->streamStage = zcss_init;
-        if (endOp == ZSTD_e_end && flushMin==0)
+        if (endOp == ZSTD_e_end && flushMin==0) {
             cctx->streamStage = zcss_init;   /* compression completed */
+            cctx->frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
+        }
         return flushMin;
     }
 #endif

From 2cf7755da7d87956b02833f49d069cf955710d2e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 12:34:41 -0700
Subject: [PATCH 069/109] fix : pledgedSrcSize correctly reset to unknown in
 "continue" mode

---
 lib/compress/zstd_compress.c | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 5334f04f5..c79ce3951 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -309,7 +309,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 #endif
 
     case ZSTD_p_contentSizeFlag :
-        DEBUGLOG(2, "set content size flag = %u", (value>0));
+        DEBUGLOG(5, "set content size flag = %u", (value>0));
         /* Content size written in frame header _when known_ (default:1) */
         cctx->requestedParams.fParams.contentSizeFlag = value>0;
         return 0;
@@ -372,7 +372,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
 ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
-    DEBUGLOG(2, " setting pledgedSrcSize to %u", (U32)pledgedSrcSize);
+    DEBUGLOG(5, " setting pledgedSrcSize to %u", (U32)pledgedSrcSize);
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     cctx->frameContentSize = pledgedSrcSize;
     return 0;
@@ -529,9 +529,14 @@ static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
 static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 frameContentSize)
 {
     U32 const end = (U32)(cctx->nextSrc - cctx->base);
+    DEBUGLOG(5, "continue mode");
     cctx->appliedParams = params;
     cctx->frameContentSize = frameContentSize;
     cctx->consumedSrcSize = 0;
+    if (frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN)
+        cctx->appliedParams.fParams.contentSizeFlag = 0;
+    DEBUGLOG(5, "content size : %u ; flag : %u",
+        (U32)frameContentSize, cctx->appliedParams.fParams.contentSizeFlag);
     cctx->lowLimit = end;
     cctx->dictLimit = end;
     cctx->nextToUpdate = end+1;
@@ -625,6 +630,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->consumedSrcSize = 0;
         if (frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN)
             zc->appliedParams.fParams.contentSizeFlag = 0;
+        DEBUGLOG(5, "content size : %u ; flag : %u",
+            (U32)frameContentSize, zc->appliedParams.fParams.contentSizeFlag);
         zc->blockSize = blockSize;
 
         XXH64_reset(&zc->xxhState, 0);
@@ -3200,9 +3207,9 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
     endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
     if (ZSTD_isError(endResult)) return endResult;
     if (cctx->appliedParams.fParams.contentSizeFlag) {  /* control src size */
-        DEBUGLOG(2, "end of frame : controlling src size");
+        DEBUGLOG(5, "end of frame : controlling src size");
         if (cctx->frameContentSize != cctx->consumedSrcSize) {
-            DEBUGLOG(2, "error : pledgedSrcSize = %u, while realSrcSize = %u",
+            DEBUGLOG(5, "error : pledgedSrcSize = %u, while realSrcSize = %u",
                 (U32)cctx->frameContentSize, (U32)cctx->consumedSrcSize);
             return ERROR(srcSize_wrong);
     }   }
@@ -3382,9 +3389,9 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
                             + cctxSize;
     ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
     void* ptr;
-    DEBUGLOG(2, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
+    DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
     if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
-    DEBUGLOG(2, "(workspaceSize < neededSize) : (%u < %u) => %u",
+    DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u",
         (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
     if (workspaceSize < neededSize) return NULL;
 

From a0ba849fe6b8914ee8042714739ee33dad6b653c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 13:29:17 -0700
Subject: [PATCH 070/109] changed frameContentSize field to
 pledgedSrcSizePlusOne

pledgedSrcSize is proper : it's a promise, not yet fulfilled.
It will be controlled at the end.

PlusOne is meant to have 0 (default) == unknown
---
 lib/compress/zstd_compress.c | 45 ++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index c79ce3951..97d19ba2f 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -98,7 +98,7 @@ struct ZSTD_CCtx_s {
     void* workSpace;
     size_t workSpaceSize;
     size_t blockSize;
-    U64 frameContentSize;
+    U64 pledgedSrcSizePlusOne;  /* this way, 0 (default) == unknown */
     U64 consumedSrcSize;
     XXH64_state_t xxhState;
     ZSTD_customMem customMem;
@@ -152,6 +152,8 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
     if (!cctx) return NULL;
     cctx->customMem = customMem;
     cctx->compressionLevel = ZSTD_CLEVEL_DEFAULT;
+    ZSTD_STATIC_ASSERT(zcss_init==0);
+    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==-(1ULL));
     return cctx;
 }
 
@@ -233,7 +235,7 @@ static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
 {
     if (cctx->compressionLevel==ZSTD_CLEVEL_CUSTOM) return;
     cctx->requestedParams.cParams = ZSTD_getCParams(cctx->compressionLevel,
-                                                    cctx->frameContentSize, 0);
+                                            cctx->pledgedSrcSizePlusOne-1, 0);
     cctx->compressionLevel = ZSTD_CLEVEL_CUSTOM;
 }
 
@@ -374,7 +376,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
 {
     DEBUGLOG(5, " setting pledgedSrcSize to %u", (U32)pledgedSrcSize);
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
-    cctx->frameContentSize = pledgedSrcSize;
+    cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
     return 0;
 }
 
@@ -421,7 +423,7 @@ size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 void ZSTD_CCtx_reset(ZSTD_CCtx* cctx)
 {
     cctx->streamStage = zcss_init;
-    cctx->frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
+    cctx->pledgedSrcSizePlusOne = 0;
     cctx->cdict = NULL;
 }
 
@@ -531,7 +533,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
     U32 const end = (U32)(cctx->nextSrc - cctx->base);
     DEBUGLOG(5, "continue mode");
     cctx->appliedParams = params;
-    cctx->frameContentSize = frameContentSize;
+    cctx->pledgedSrcSizePlusOne = frameContentSize+1;
     cctx->consumedSrcSize = 0;
     if (frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN)
         cctx->appliedParams.fParams.contentSizeFlag = 0;
@@ -554,7 +556,7 @@ typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e;
 /*! ZSTD_resetCCtx_internal() :
     note : `params` are assumed fully validated at this stage */
 static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
-                                      ZSTD_parameters params, U64 frameContentSize,
+                                      ZSTD_parameters params, U64 pledgedSrcSize,
                                       ZSTD_compResetPolicy_e const crp,
                                       ZSTD_buffered_policy_e const zbuff)
 {
@@ -565,7 +567,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
             DEBUGLOG(5, "ZSTD_equivalentParams()==1");
             zc->fseCTables_ready = 0;
             zc->hufCTable_repeatMode = HUF_repeat_none;
-            return ZSTD_continueCCtx(zc, params, frameContentSize);
+            return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
     }   }
 
     {   size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
@@ -626,9 +628,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
         /* init params */
         zc->appliedParams = params;
-        zc->frameContentSize = frameContentSize;
+        zc->pledgedSrcSizePlusOne = pledgedSrcSize+1;
         zc->consumedSrcSize = 0;
-        if (frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN)
+        if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)
             zc->appliedParams.fParams.contentSizeFlag = 0;
         DEBUGLOG(5, "content size : %u ; flag : %u",
             (U32)frameContentSize, zc->appliedParams.fParams.contentSizeFlag);
@@ -2873,7 +2875,8 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
     if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong);   /* missing init (ZSTD_compressBegin) */
 
     if (frame && (cctx->stage==ZSTDcs_init)) {
-        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, cctx->frameContentSize, cctx->dictID);
+        fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams,
+                                cctx->pledgedSrcSizePlusOne-1, cctx->dictID);
         if (ZSTD_isError(fhSize)) return fhSize;
         dstCapacity -= fhSize;
         dst = (char*)dst + fhSize;
@@ -3208,7 +3211,7 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
     if (ZSTD_isError(endResult)) return endResult;
     if (cctx->appliedParams.fParams.contentSizeFlag) {  /* control src size */
         DEBUGLOG(5, "end of frame : controlling src size");
-        if (cctx->frameContentSize != cctx->consumedSrcSize) {
+        if (cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1) {
             DEBUGLOG(5, "error : pledgedSrcSize = %u, while realSrcSize = %u",
                 (U32)cctx->frameContentSize, (U32)cctx->consumedSrcSize);
             return ERROR(srcSize_wrong);
@@ -3697,7 +3700,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                         DEBUGLOG(5, "Frame completed directly in outBuffer");
                         someMoreWork = 0;
                         zcs->streamStage = zcss_init;
-                        zcs->frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
+                        zcs->pledgedSrcSizePlusOne = 0;
                     }
                     break;
                 }
@@ -3724,7 +3727,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                     DEBUGLOG(5, "Frame completed on flush");
                     someMoreWork = 0;
                     zcs->streamStage = zcss_init;
-                    zcs->frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
+                    zcs->pledgedSrcSizePlusOne = 0;
                     break;
                 }
                 zcs->streamStage = zcss_load;
@@ -3782,27 +3785,29 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         ZSTD_parameters params = cctx->requestedParams;
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
-                                    cctx->frameContentSize, 0 /* dictSize */);
+                                    cctx->pledgedSrcSizePlusOne+1, 0 /* dictSize */);
 
 #ifdef ZSTD_MULTITHREAD
         if (cctx->nbThreads > 1) {
-            CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, NULL, 0, cctx->cdict, params, cctx->frameContentSize) );
+            CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, NULL, 0, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
             cctx->streamStage = zcss_load;
         } else
 #endif
         {
-            CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->frameContentSize) );
+            CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->pledgedSrcSizePlusOne+1) );
     }   }
 
 #ifdef ZSTD_MULTITHREAD
     if (cctx->nbThreads > 1) {
-        DEBUGLOG(5, "calling ZSTDMT_compressStream_generic(%i,...)", endOp);
         size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
         DEBUGLOG(5, "ZSTDMT result : %u", (U32)flushMin);
-        if (ZSTD_isError(flushMin)) cctx->streamStage = zcss_init;
-        if (endOp == ZSTD_e_end && flushMin==0) {
+        if (ZSTD_isError(flushMin)) {
+            cctx->streamStage = zcss_init;
+            cctx->pledgedSrcSizePlusOne = 0;
+        }
+        if (endOp == ZSTD_e_end && flushMin == 0) {
             cctx->streamStage = zcss_init;   /* compression completed */
-            cctx->frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
+            cctx->pledgedSrcSizePlusOne = 0;
         }
         return flushMin;
     }

From b26728c9c8c01b9546a37f9e3a2bdfadcf142fce Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 14:00:46 -0700
Subject: [PATCH 071/109] added ZSTD_startNewCompression()

---
 lib/compress/zstd_compress.c | 40 ++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 97d19ba2f..ad3736ced 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -419,11 +419,17 @@ size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
     return ERROR(compressionParameter_unsupported);
 }
 
-
-void ZSTD_CCtx_reset(ZSTD_CCtx* cctx)
+static void ZSTD_startNewCompression(ZSTD_CCtx* cctx)
 {
     cctx->streamStage = zcss_init;
     cctx->pledgedSrcSizePlusOne = 0;
+}
+
+/*! ZSTD_CCtx_reset() :
+ *  Also dumps dictionary */
+void ZSTD_CCtx_reset(ZSTD_CCtx* cctx)
+{
+    ZSTD_startNewCompression(cctx);
     cctx->cdict = NULL;
 }
 
@@ -528,17 +534,17 @@ static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
 
 /*! ZSTD_continueCCtx() :
     reuse CCtx without reset (note : requires no dictionary) */
-static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 frameContentSize)
+static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 pledgedSrcSize)
 {
     U32 const end = (U32)(cctx->nextSrc - cctx->base);
     DEBUGLOG(5, "continue mode");
     cctx->appliedParams = params;
-    cctx->pledgedSrcSizePlusOne = frameContentSize+1;
+    cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
     cctx->consumedSrcSize = 0;
-    if (frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN)
+    if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)
         cctx->appliedParams.fParams.contentSizeFlag = 0;
-    DEBUGLOG(5, "content size : %u ; flag : %u",
-        (U32)frameContentSize, cctx->appliedParams.fParams.contentSizeFlag);
+    DEBUGLOG(5, "pledged content size : %u ; flag : %u",
+        (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag);
     cctx->lowLimit = end;
     cctx->dictLimit = end;
     cctx->nextToUpdate = end+1;
@@ -632,8 +638,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->consumedSrcSize = 0;
         if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)
             zc->appliedParams.fParams.contentSizeFlag = 0;
-        DEBUGLOG(5, "content size : %u ; flag : %u",
-            (U32)frameContentSize, zc->appliedParams.fParams.contentSizeFlag);
+        DEBUGLOG(5, "pledged content size : %u ; flag : %u",
+            (U32)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag);
         zc->blockSize = blockSize;
 
         XXH64_reset(&zc->xxhState, 0);
@@ -3699,8 +3705,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                     if (zcs->frameEnded) {
                         DEBUGLOG(5, "Frame completed directly in outBuffer");
                         someMoreWork = 0;
-                        zcs->streamStage = zcss_init;
-                        zcs->pledgedSrcSizePlusOne = 0;
+                        ZSTD_startNewCompression(zcs);
                     }
                     break;
                 }
@@ -3726,8 +3731,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 if (zcs->frameEnded) {
                     DEBUGLOG(5, "Frame completed on flush");
                     someMoreWork = 0;
-                    zcs->streamStage = zcss_init;
-                    zcs->pledgedSrcSizePlusOne = 0;
+                    ZSTD_startNewCompression(zcs);
                     break;
                 }
                 zcs->streamStage = zcss_load;
@@ -3801,13 +3805,9 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     if (cctx->nbThreads > 1) {
         size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
         DEBUGLOG(5, "ZSTDMT result : %u", (U32)flushMin);
-        if (ZSTD_isError(flushMin)) {
-            cctx->streamStage = zcss_init;
-            cctx->pledgedSrcSizePlusOne = 0;
-        }
-        if (endOp == ZSTD_e_end && flushMin == 0) {
-            cctx->streamStage = zcss_init;   /* compression completed */
-            cctx->pledgedSrcSizePlusOne = 0;
+        if ( ZSTD_isError(flushMin)
+          || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
+            ZSTD_startNewCompression(cctx);
         }
         return flushMin;
     }

From 944be54774f03a5aa82641c970fa5bdc66e39853 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 14:04:30 -0700
Subject: [PATCH 072/109] fixed attempt : minor Visual sign conversion warning

---
 lib/compress/zstd_compress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ad3736ced..8993e7d75 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -153,7 +153,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
     cctx->customMem = customMem;
     cctx->compressionLevel = ZSTD_CLEVEL_DEFAULT;
     ZSTD_STATIC_ASSERT(zcss_init==0);
-    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==-(1ULL));
+    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(U64)(-(1ULL)));
     return cctx;
 }
 

From d3de3d51a37d5587e62ad1030daa08c775be6749 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 16:51:33 -0700
Subject: [PATCH 073/109] fix attempt 2 : Visual sign conversion warning

---
 lib/compress/zstd_compress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 8993e7d75..937fd3107 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -153,7 +153,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
     cctx->customMem = customMem;
     cctx->compressionLevel = ZSTD_CLEVEL_DEFAULT;
     ZSTD_STATIC_ASSERT(zcss_init==0);
-    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(U64)(-(1ULL)));
+    ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1));
     return cctx;
 }
 

From aee916e37c159b2fc4c8e273597d36e0d6307797 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 17:01:46 -0700
Subject: [PATCH 074/109] fixed +/-1 error for pledgedSrcSizePlusOne

---
 lib/compress/zstd_compress.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 937fd3107..6286c5aa6 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3219,7 +3219,7 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
         DEBUGLOG(5, "end of frame : controlling src size");
         if (cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1) {
             DEBUGLOG(5, "error : pledgedSrcSize = %u, while realSrcSize = %u",
-                (U32)cctx->frameContentSize, (U32)cctx->consumedSrcSize);
+                (U32)cctx->pledgedSrcSizePlusOne-1, (U32)cctx->consumedSrcSize);
             return ERROR(srcSize_wrong);
     }   }
     return cSize + endResult;
@@ -3789,7 +3789,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         ZSTD_parameters params = cctx->requestedParams;
         if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
             params.cParams = ZSTD_getCParams(cctx->compressionLevel,
-                                    cctx->pledgedSrcSizePlusOne+1, 0 /* dictSize */);
+                                    cctx->pledgedSrcSizePlusOne-1, 0 /* dictSize */);
 
 #ifdef ZSTD_MULTITHREAD
         if (cctx->nbThreads > 1) {
@@ -3798,7 +3798,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         } else
 #endif
         {
-            CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->pledgedSrcSizePlusOne+1) );
+            CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->pledgedSrcSizePlusOne-1) );
     }   }
 
 #ifdef ZSTD_MULTITHREAD

From 381e66cfbd3b59c34fb006e3ca3d70a709c6ca48 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 17:29:35 -0700
Subject: [PATCH 075/109] added ZSTD_clampCParams()

now ZSTD_adjustCParams() is always successful,
it always produces a valid CParams
---
 lib/compress/zstd_compress.c   | 37 ++++++++++++++++++++++++++++------
 lib/compress/zstdmt_compress.c |  4 ++--
 lib/zstd.h                     |  4 ++--
 3 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 6286c5aa6..f848c088e 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -433,8 +433,8 @@ void ZSTD_CCtx_reset(ZSTD_CCtx* cctx)
     cctx->cdict = NULL;
 }
 
-/** ZSTD_checkParams() :
-    ensure param values remain within authorized range.
+/** ZSTD_checkCParams() :
+    control CParam values remain within authorized range.
     @return : 0, or an error code if one value is beyond authorized range */
 size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
 {
@@ -448,6 +448,25 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
     return 0;
 }
 
+/** ZSTD_clampCParams() :
+ *  make CParam values within valid range.
+ *  @return : valid CParams */
+static ZSTD_compressionParameters ZSTD_clampCParams(ZSTD_compressionParameters cParams)
+{
+#   define CLAMP(val,min,max) {      \
+        if (valmax) val=max;   \
+    }
+    CLAMP(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+    CLAMP(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
+    CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
+    CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
+    CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
+    CLAMP(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
+    if ((U32)(cParams.strategy) > (U32)ZSTD_btultra) cParams.strategy = ZSTD_btultra;
+    return cParams;
+}
+
 /** ZSTD_cycleLog() :
  *  condition for correct operation : hashLog > 1 */
 static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
@@ -456,14 +475,15 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
     return hashLog - btScale;
 }
 
-/** ZSTD_adjustCParams() :
+/** ZSTD_adjustCParams_internal() :
     optimize `cPar` for a given input (`srcSize` and `dictSize`).
     mostly downsizing to reduce memory consumption and initialization.
     Both `srcSize` and `dictSize` are optional (use 0 if unknown),
     but if both are 0, no optimization can be done.
     Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
-ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
+ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
 {
+    assert(ZSTD_checkCParams(cPar)==0);
     if (srcSize+dictSize == 0) return cPar;   /* no size information available : no adjustment */
 
     /* resize params, to use less memory when necessary */
@@ -483,6 +503,12 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
     return cPar;
 }
 
+ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
+{
+    cPar = ZSTD_clampCParams(cPar);
+    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize);
+}
+
 
 size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
 {
@@ -3989,8 +4015,7 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
         if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
         if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
     }
-    cp = ZSTD_adjustCParams(cp, srcSizeHint, dictSize);
-    return cp;
+    return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize);   /* no need to ensure initial CParams validity */
 }
 
 /*! ZSTD_getParams() :
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index a4b4f5160..42e31ca18 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -190,7 +190,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
     cctxPool->availCCtx = 1;   /* at least one cctx for single-thread mode */
     cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem);
     if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
-    DEBUGLOG(1, "cctxPool created, with %u threads", nbThreads);
+    DEBUGLOG(3, "cctxPool created, with %u threads", nbThreads);
     return cctxPool;
 }
 
@@ -458,7 +458,7 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
     size_t frameStartPos = 0, dstBufferPos = 0;
 
     DEBUGLOG(3, "windowLog : %2u => chunkTargetSize : %u bytes  ", params.cParams.windowLog, (U32)chunkTargetSize);
-    DEBUGLOG(2, "nbChunks  : %2u   (chunkSize : %u bytes)   ", nbChunks, (U32)avgChunkSize);
+    DEBUGLOG(3, "nbChunks  : %2u   (chunkSize : %u bytes)   ", nbChunks, (U32)avgChunkSize);
     params.fParams.contentSizeFlag = 1;
 
     if (nbChunks==1) {   /* fallback to single-thread mode */
diff --git a/lib/zstd.h b/lib/zstd.h
index 1e73b5277..928abe240 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -589,8 +589,8 @@ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long l
 ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
 
 /*! ZSTD_adjustCParams() :
-*   optimize params for a given `srcSize` and `dictSize`.
-*   both values are optional, select `0` if unknown. */
+ *  optimize params for a given `srcSize` and `dictSize`.
+ *  both values are optional, select `0` if unknown. */
 ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 
 /*! ZSTD_compress_advanced() :

From 01743a36e78950a4113dd12f7fd79ab5009c1a31 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 16 Jun 2017 17:56:41 -0700
Subject: [PATCH 076/109] fuzzer tests for new API

---
 NEWS                 |   2 +
 doc/zstd_manual.html |   9 +-
 tests/Makefile       |  16 ++-
 tests/zstreamtest.c  | 334 +++++++++++++++++++++++++++++++++++++++++--
 4 files changed, 339 insertions(+), 22 deletions(-)

diff --git a/NEWS b/NEWS
index 9f1298126..3f9b4b220 100644
--- a/NEWS
+++ b/NEWS
@@ -1,9 +1,11 @@
 v1.3.0
 cli : changed : `-t *` does no longer stop after a decompression error
+API : added : ZSTD_versionString()
 API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
 API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
 API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
 API exp : changed : strongest strategy renamed ZSTD_btultra
+API exp : clarified presentation of memory estimation / measurement functions.
 new : contrib/seekable_format, demo and API, by Sean Purcell
 changed : contrib/linux-kernel, updated version and license, by Nick Terrell
 
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 523c49bb4..d108acaa4 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -581,11 +581,11 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
                               * More threads improve speed, but also increase memory usage.
                               * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
                               * Special: value 0 means "do not change nbThreads" */
-    ZSTDMT_p_jobSize,        /* Size of a compression job. Each compression job is completed in parallel.
+    ZSTD_p_jobSize,          /* Size of a compression job. Each compression job is completed in parallel.
                               * 0 means default, which is dynamically determined based on compression parameters.
                               * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
                               * The minimum size is automatically and transparently enforced */
-    ZSTDMT_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job.
+    ZSTD_p_overlapSizeLog,   /* Size of previous input reloaded at the beginning of each job.
                               * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
 
     /* advanced parameters - may not remain available after API update */
@@ -691,11 +691,12 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
  
 


 
-
size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
+
void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
 
  Return a CCtx to clean state.
   Useful after an error, or to interrupt an ongoing compression job and start a new one.
-  It's possible to modify compression parameters after a reset.
   Any internal data not yet flushed is cancelled.
+  Dictionary (if any) is dropped, next compression starts with srcSize==unknown by default.
+  It's possible to modify compression parameters after a reset.
  
 


 
diff --git a/tests/Makefile b/tests/Makefile
index 0ee182fa8..c116404b8 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -30,14 +30,14 @@ DEBUGFLAGS= -g -DZSTD_DEBUG=1
 CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
             -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
 CFLAGS   ?= -O3
-CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-            -Wstrict-prototypes -Wundef -Wformat-security \
-            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls \
-            $(DEBUGFLAGS)
-CFLAGS  += $(MOREFLAGS)
-FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+            -Wstrict-prototypes -Wundef -Wformat-security                   \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
+            -Wredundant-decls
+CFLAGS   += $(DEBUGFLAGS)
+CFLAGS   += $(MOREFLAGS)
+FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
@@ -321,6 +321,8 @@ test-zbuff32: zbufftest32
 
 test-zstream: zstreamtest
 	$(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./zstreamtest --mt $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
+	$(QEMU_SYS) ./zstreamtest --newapi $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
 
 test-zstream32: zstreamtest32
 	$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 17772856e..424d1cf63 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -25,6 +25,7 @@
 #include         /* fgets, sscanf */
 #include          /* clock_t, clock() */
 #include        /* strcmp */
+#include        /* assert */
 #include "mem.h"
 #define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
 #include "zstd.h"         /* ZSTD_compressBound */
@@ -646,8 +647,22 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
 
 #define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
 
-#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
-                         DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }
+#define CHECK(cond, ...) {                                   \
+    if (cond) {                                              \
+        DISPLAY("Error => ");                                \
+        DISPLAY(__VA_ARGS__);                                \
+        DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); \
+        goto _output_error;                                  \
+}   }
+
+#define CHECK_Z(f) {                                         \
+    size_t const err = f;                                    \
+    if (ZSTD_isError(err)) {                                 \
+        DISPLAY("Error => %s : %s ",                         \
+                #f, ZSTD_getErrorName(err));                 \
+        DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); \
+        goto _output_error;                                  \
+}   }
 
 static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
 {
@@ -739,8 +754,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
             if (maxTestSize >= srcBufferSize)
                 maxTestSize = srcBufferSize-1;
             {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
-                size_t const resetError = ZSTD_resetCStream(zc, pledgedSrcSize);
-                CHECK(ZSTD_isError(resetError), "ZSTD_resetCStream error : %s", ZSTD_getErrorName(resetError));
+                CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) );
             }
         } else {
             U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
@@ -760,9 +774,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
                 ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
                 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
-                {   size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize);
-                    CHECK (ZSTD_isError(initError),"ZSTD_initCStream_advanced error : %s", ZSTD_getErrorName(initError));
-        }   }   }
+                CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
+        }   }
 
         /* multi-segments compression test */
         XXH64_reset(&xxhState, 0);
@@ -1142,6 +1155,289 @@ _output_error:
 }
 
 
+/* Tests for ZSTD_compressGeneric() API */
+static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
+{
+    static const U32 maxSrcLog = 24;
+    static const U32 maxSampleLog = 19;
+    size_t const srcBufferSize = (size_t)1<= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests); }
+        else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
+        FUZ_rand(&coreSeed);
+        lseed = coreSeed ^ prime32;
+
+        /* states full reset (deliberately not synchronized) */
+        /* some issues can only happen when reusing states */
+        if ((FUZ_rand(&lseed) & 0xFF) == 131) {
+            DISPLAYLEVEL(5, "Creating new context \n");
+            ZSTD_freeCCtx(zc);
+            zc = ZSTD_createCCtx();
+            CHECK(zc==NULL, "allocation error");
+            resetAllowed=0;
+        }
+        if ((FUZ_rand(&lseed) & 0xFF) == 132) {
+            ZSTD_freeDStream(zd);
+            zd = ZSTD_createDStream();
+            CHECK(zd==NULL, "allocation error");
+            ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
+        }
+
+        /* srcBuffer selection [0-4] */
+        {   U32 buffNb = FUZ_rand(&lseed) & 0x7F;
+            if (buffNb & 7) buffNb=2;   /* most common : compressible (P) */
+            else {
+                buffNb >>= 3;
+                if (buffNb & 7) {
+                    const U32 tnb[2] = { 1, 3 };   /* barely/highly compressible */
+                    buffNb = tnb[buffNb >> 3];
+                } else {
+                    const U32 tnb[2] = { 0, 4 };   /* not compressible / sparse */
+                    buffNb = tnb[buffNb >> 3];
+            }   }
+            srcBuffer = cNoiseBuffer[buffNb];
+        }
+
+        /* compression init */
+        if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
+            && oldTestLog /* at least one test happened */ && resetAllowed) {
+            maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
+            if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
+            CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );
+            {   int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, compressionLevel) );
+            }
+        } else {
+            U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
+            U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
+            U32 const cLevel = (FUZ_rand(&lseed) %
+                               (ZSTD_maxCLevel() -
+                               (MAX(testLog, dictLog) / cLevelLimiter))) +
+                               1;
+            maxTestSize = FUZ_rLogLength(&lseed, testLog);
+            oldTestLog = testLog;
+            /* random dictionary selection */
+            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
+            {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
+                dict = srcBuffer + dictStart;
+            }
+            {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
+                ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
+
+                /* mess with compression parameters */
+                cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
+                cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
+                cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
+                cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
+                cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1;
+                cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((FUZ_rand(&lseed) & 127) * 128)));
+                cParams = ZSTD_adjustCParams(cParams, 0, 0);
+
+                /* to do : check everything works properly when only some parameters are set and others are not */
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog) );
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_hashLog, cParams.hashLog) );
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_chainLog, cParams.chainLog) );
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_searchLog, cParams.searchLog) );
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_minMatch, cParams.searchLength) );
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
+
+                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
+
+                /* to do : check that cParams are blocked after loading non-NULL dictionary */
+
+                /* mess with frame parameters */
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) );
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1) );
+                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_contentSizeFlag, pledgedSrcSize>0) );
+                CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );   /* <== to test : must work also when pledgedSrcSize not set ! */
+                DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
+
+                /* multi-threading parameters */
+                {   U32 const nbThreads = (FUZ_rand(&lseed) & 3) + 1;
+                    CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, nbThreads) );
+                    if (nbThreads > 1) {
+                        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10) );
+                        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) );
+        }   }   }   }
+
+        /* multi-segments compression test */
+        XXH64_reset(&xxhState, 0);
+        {   ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
+            U32 n;
+            for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
+                /* compress random chunks into randomly sized dst buffers */
+                {   size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
+                    size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
+                    ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
+                    outBuff.size = outBuff.pos + dstBuffSize;
+
+                    CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_continue) );
+                    DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n",
+                        (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
+
+                    XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
+                    memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
+                    totalTestSize += inBuff.pos;
+                }
+
+                /* random flush operation, to mess around */
+                if ((FUZ_rand(&lseed) & 15) == 0) {
+                    ZSTD_inBuffer inBuff = { NULL, 0, 0 };
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
+                    outBuff.size = outBuff.pos + adjustedDstSize;
+                    DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (U32)adjustedDstSize);
+                    CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_flush) );
+            }   }
+
+            /* final frame epilogue */
+            {   size_t remainingToFlush = (size_t)(-1);
+                while (remainingToFlush) {
+                    ZSTD_inBuffer inBuff = { NULL, 0, 0 };
+                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                    size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
+                    outBuff.size = outBuff.pos + adjustedDstSize;
+                    DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize);
+                    remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
+                    CHECK(ZSTD_isError(remainingToFlush), "ZSTD_compress_generic w/ ZSTD_e_end error : %s", ZSTD_getErrorName(remainingToFlush));
+            }   }
+            crcOrig = XXH64_digest(&xxhState);
+            cSize = outBuff.pos;
+            DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize);
+        }
+
+        /* multi - fragments decompression test */
+        if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
+            CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
+        } else {
+            ZSTD_initDStream_usingDict(zd, dict, dictSize);
+        }
+        {   size_t decompressionResult = 1;
+            ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
+            ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
+            for (totalGenSize = 0 ; decompressionResult ; ) {
+                size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
+                inBuff.size = inBuff.pos + readCSrcSize;
+                outBuff.size = inBuff.pos + dstBuffSize;
+                DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize);
+                decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+                CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
+                DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
+            }
+            CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
+            CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
+            {   U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
+                if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
+                CHECK (crcDest!=crcOrig, "decompressed data corrupted");
+        }   }
+
+        /*=====   noisy/erroneous src decompression test   =====*/
+
+        /* add some noise */
+        {   U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
+            U32 nn; for (nn=0; nn
Date: Fri, 16 Jun 2017 18:16:27 -0700
Subject: [PATCH 077/109] removed macro CLAMP from decodecorpus

never used,
and would duplicate with CLAMP from zstd_compress.c
---
 tests/decodecorpus.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index f7b3c854f..940e3d8a1 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -73,8 +73,6 @@ static clock_t clockSpan(clock_t cStart)
 /*-*******************************************************
 *  Random function
 *********************************************************/
-#define CLAMP(x, a, b) ((x) < (a) ? (a) : ((x) > (b) ? (b) : (x)))
-
 static unsigned RAND(unsigned* src)
 {
 #define RAND_rotl32(x,r) ((x << r) | (x >> (32 - r)))

From 8dee0ec99e2259e863afaa14ecf5f723e6518cba Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 18 Jun 2017 23:25:15 -0700
Subject: [PATCH 078/109] new api fuzzer tests : compression parameters are
 randomly set

---
 tests/zstreamtest.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 424d1cf63..e087dd722 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1197,7 +1197,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
     RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
     RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
     memset(copyBuffer, 0x65, copyBufferSize);                             /* make copyBuffer considered initialized */
-    ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
+    ZSTD_initDStream_usingDict(zd, NULL, 0);   /* ensure at least one init */
 
     /* catch up testNb */
     for (testNb=1; testNb < startTest; testNb++)
@@ -1285,13 +1285,12 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((FUZ_rand(&lseed) & 127) * 128)));
                 cParams = ZSTD_adjustCParams(cParams, 0, 0);
 
-                /* to do : check everything works properly when only some parameters are set and others are not */
-                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog) );
-                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_hashLog, cParams.hashLog) );
-                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_chainLog, cParams.chainLog) );
-                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_searchLog, cParams.searchLog) );
-                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_minMatch, cParams.searchLength) );
-                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_hashLog, cParams.hashLog) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_chainLog, cParams.chainLog) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_searchLog, cParams.searchLog) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_minMatch, cParams.searchLength) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
 
                 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
 
@@ -1416,7 +1415,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                     /* No forward progress possible */
                     if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
     }   }   }   }
-    DISPLAY("\r%u fuzzer tests completed   \n", testNb);
+    DISPLAY("\r%u fuzzer tests completed   \n", testNb-1);
 
 _cleanup:
     ZSTD_freeCCtx(zc);

From ea3630a88997b88f9f17c6fdf637ed15e44b69d8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 18 Jun 2017 23:31:55 -0700
Subject: [PATCH 079/109] new api fuzzer set : dictionary is randomly set

---
 tests/zstreamtest.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index e087dd722..0785aecb4 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1292,6 +1292,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_minMatch, cParams.searchLength) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
 
+                if (FUZ_rand(&lseed) & 1) { dict=NULL; dictSize=0; }
                 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
 
                 /* to do : check that cParams are blocked after loading non-NULL dictionary */

From f5deae8a672754f0a1b0785c9fbec78fb9deca76 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 18 Jun 2017 23:41:38 -0700
Subject: [PATCH 080/109] new api fuzzer tests : frame parameters are randonly
 set

---
 tests/zstreamtest.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 0785aecb4..d5b50fd5c 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1282,7 +1282,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1;
-                cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((FUZ_rand(&lseed) & 127) * 128)));
+                cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
                 cParams = ZSTD_adjustCParams(cParams, 0, 0);
 
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog) );
@@ -1293,15 +1293,15 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
 
                 if (FUZ_rand(&lseed) & 1) { dict=NULL; dictSize=0; }
-                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
+                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );   /* unconditionally set, to be sync with decoder (could be improved) */
 
                 /* to do : check that cParams are blocked after loading non-NULL dictionary */
 
                 /* mess with frame parameters */
-                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) );
-                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1) );
-                CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_contentSizeFlag, pledgedSrcSize>0) );
-                CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );   /* <== to test : must work also when pledgedSrcSize not set ! */
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1) );
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
                 DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
 
                 /* multi-threading parameters */

From ed1d0391272137b5a53b3e2c32ac4a19787d8116 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 19 Jun 2017 11:07:33 -0700
Subject: [PATCH 081/109] newapi fuzzer tests : random flush orders in main
 loop

---
 tests/zstreamtest.c | 64 ++++++++++++++++++++++-----------------------
 1 file changed, 31 insertions(+), 33 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index d5b50fd5c..09aaae54d 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -729,8 +729,16 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
 
         /* states full reset (deliberately not synchronized) */
         /* some issues can only happen when reusing states */
-        if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); resetAllowed=0; }
-        if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */ }
+        if ((FUZ_rand(&lseed) & 0xFF) == 131) {
+            ZSTD_freeCStream(zc);
+            zc = ZSTD_createCStream();
+            resetAllowed=0;
+        }
+        if ((FUZ_rand(&lseed) & 0xFF) == 132) {
+            ZSTD_freeDStream(zd);
+            zd = ZSTD_createDStream();
+            ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
+        }
 
         /* srcBuffer selection [0-4] */
         {   U32 buffNb = FUZ_rand(&lseed) & 0x7F;
@@ -791,8 +799,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
                     ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
                     outBuff.size = outBuff.pos + dstBuffSize;
 
-                    { size_t const compressionError = ZSTD_compressStream(zc, &outBuff, &inBuff);
-                      CHECK (ZSTD_isError(compressionError), "compression error : %s", ZSTD_getErrorName(compressionError)); }
+                    CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
 
                     XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
                     memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
@@ -804,9 +811,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
                     size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
                     size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
                     outBuff.size = outBuff.pos + adjustedDstSize;
-                    {   size_t const flushError = ZSTD_flushStream(zc, &outBuff);
-                        CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError));
-            }   }   }
+                    CHECK_Z( ZSTD_flushStream(zc, &outBuff) );
+            }   }
 
             /* final frame epilogue */
             {   size_t remainingToFlush = (size_t)(-1);
@@ -1308,8 +1314,9 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 {   U32 const nbThreads = (FUZ_rand(&lseed) & 3) + 1;
                     CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, nbThreads) );
                     if (nbThreads > 1) {
+                        U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
                         CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10) );
-                        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) );
+                        CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog)) );
         }   }   }   }
 
         /* multi-segments compression test */
@@ -1318,32 +1325,23 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
             U32 n;
             for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
                 /* compress random chunks into randomly sized dst buffers */
-                {   size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
-                    size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
-                    size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
-                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
-                    size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
-                    ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
-                    outBuff.size = outBuff.pos + dstBuffSize;
+                size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
+                size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
+                size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
+                size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
+                ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
+                ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
+                outBuff.size = outBuff.pos + dstBuffSize;
 
-                    CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_continue) );
-                    DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n",
-                        (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
+                CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
+                DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n",
+                    (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
 
-                    XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
-                    memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
-                    totalTestSize += inBuff.pos;
-                }
-
-                /* random flush operation, to mess around */
-                if ((FUZ_rand(&lseed) & 15) == 0) {
-                    ZSTD_inBuffer inBuff = { NULL, 0, 0 };
-                    size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
-                    size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
-                    outBuff.size = outBuff.pos + adjustedDstSize;
-                    DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (U32)adjustedDstSize);
-                    CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_flush) );
-            }   }
+                XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
+                memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
+                totalTestSize += inBuff.pos;
+            }
 
             /* final frame epilogue */
             {   size_t remainingToFlush = (size_t)(-1);
@@ -1413,7 +1411,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 inBuff.size  = inBuff.pos + adjustedCSrcSize;
                 {   size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
                     if (ZSTD_isError(decompressError)) break;   /* error correctly detected */
-                    /* No forward progress possible */
+                    /* Good so far, but no more progress possible */
                     if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
     }   }   }   }
     DISPLAY("\r%u fuzzer tests completed   \n", testNb-1);

From d7a3bffba909eb5c0ebe79d3f1e457c1ec7d2da9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 19 Jun 2017 11:53:01 -0700
Subject: [PATCH 082/109] new api : setting compression parameters is refused
 if a dictionary is already loaded

---
 lib/compress/zstd_compress.c | 11 +++++++++++
 lib/zstd.h                   |  7 ++++---
 tests/zstreamtest.c          | 23 ++++++++++++++++-------
 3 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index f848c088e..3620e13d8 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -253,11 +253,15 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
     case ZSTD_p_compressionLevel :
         if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel();   /* cap max compression level */
         if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        if (cctx->cdict) return ERROR(stage_wrong);
         cctx->compressionLevel = value;
         return 0;
 
     case ZSTD_p_windowLog :
+        DEBUGLOG(5, "setting ZSTD_p_windowLog = %u (cdict:%u)",
+                    value, (cctx->cdict!=NULL));
         if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        if (cctx->cdict) return ERROR(stage_wrong);
         CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
         ZSTD_cLevelToCParams(cctx);
         cctx->requestedParams.cParams.windowLog = value;
@@ -265,6 +269,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
     case ZSTD_p_hashLog :
         if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        if (cctx->cdict) return ERROR(stage_wrong);
         CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
         ZSTD_cLevelToCParams(cctx);
         cctx->requestedParams.cParams.hashLog = value;
@@ -272,6 +277,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
     case ZSTD_p_chainLog :
         if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        if (cctx->cdict) return ERROR(stage_wrong);
         CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
         ZSTD_cLevelToCParams(cctx);
         cctx->requestedParams.cParams.chainLog = value;
@@ -279,6 +285,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
     case ZSTD_p_searchLog :
         if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        if (cctx->cdict) return ERROR(stage_wrong);
         CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
         ZSTD_cLevelToCParams(cctx);
         cctx->requestedParams.cParams.searchLog = value;
@@ -286,6 +293,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
     case ZSTD_p_minMatch :
         if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        if (cctx->cdict) return ERROR(stage_wrong);
         CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
         ZSTD_cLevelToCParams(cctx);
         cctx->requestedParams.cParams.searchLength = value;
@@ -293,6 +301,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
     case ZSTD_p_targetLength :
         if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        if (cctx->cdict) return ERROR(stage_wrong);
         CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
         ZSTD_cLevelToCParams(cctx);
         cctx->requestedParams.cParams.targetLength = value;
@@ -300,6 +309,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
 
     case ZSTD_p_compressionStrategy :
         if (value == 0) return 0;  /* special value : 0 means "don't change anything" */
+        if (cctx->cdict) return ERROR(stage_wrong);
         CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
         ZSTD_cLevelToCParams(cctx);
         cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value;
@@ -384,6 +394,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
 {
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     if (cctx->staticSize) return ERROR(memory_allocation);  /* no malloc for static CCtx */
+    DEBUGLOG(5, "load dictionary of size %u", (U32)dictSize);
     ZSTD_freeCDict(cctx->cdictLocal);  /* in case one already exists */
     if (dict==NULL || dictSize==0) {   /* no dictionary mode */
         cctx->cdictLocal = NULL;
diff --git a/lib/zstd.h b/lib/zstd.h
index 928abe240..b204b9a7b 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -528,8 +528,9 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
  *         If it needs more memory than available, it will simply error out.
  *  Note 2 : there is no corresponding "free" function.
  *           Since workspace was allocated externally, it must be freed externally too.
- *  Limitation : currently not compatible with internal CDict creation, such as
- *               ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
+ *  Limitation 1 : currently not compatible with internal CDict creation, such as
+ *                 ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
+ *  Limitation 2 : currently not compatible with multi-threading
  */
 ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
 
@@ -746,7 +747,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
  *           It's also a CPU-heavy operation, with non-negligible impact on latency.
  *  Note 3 : Dictionary will be used for all future compression jobs.
  *           To return to "no-dictionary" situation, load a NULL dictionary */
-ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);   /* Not ready yet ! */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 
 /*! ZSTD_CCtx_refPrefix() :
  *  Reference a prefix (content-only dictionary) to bootstrap next compression job.
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 09aaae54d..57993fcbb 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1161,7 +1161,7 @@ _output_error:
 }
 
 
-/* Tests for ZSTD_compressGeneric() API */
+/* Tests for ZSTD_compress_generic() API */
 static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
 {
     static const U32 maxSrcLog = 24;
@@ -1181,7 +1181,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
     ZSTD_DStream* zd = ZSTD_createDStream();   /* will be reset sometimes */
     ZSTD_DStream* const zd_noise = ZSTD_createDStream();
     clock_t const startClock = clock();
-    const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
+    const BYTE* dict = NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
     int const cLevelLimiter = bigTests ? 3 : 2;
@@ -1278,11 +1278,13 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
             dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                 dict = srcBuffer + dictStart;
+                if (!dictSize) dict=NULL;
             }
             {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
                 ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
 
                 /* mess with compression parameters */
+                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );   /* always cancel previous dict, to make user it's possible to pass compression parameters */
                 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
@@ -1298,10 +1300,15 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_minMatch, cParams.searchLength) );
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
 
-                if (FUZ_rand(&lseed) & 1) { dict=NULL; dictSize=0; }
-                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );   /* unconditionally set, to be sync with decoder (could be improved) */
+                /* unconditionally set, to be sync with decoder */
+                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
 
-                /* to do : check that cParams are blocked after loading non-NULL dictionary */
+                if (dict && dictSize) {
+                    /* test that compression parameters are correctly rejected after setting a dictionary */
+                    DISPLAYLEVEL(5, "setting windowLog while dict!=NULL \n");
+                    size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1) ;
+                    CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed");
+                }
 
                 /* mess with frame parameters */
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) );
@@ -1352,7 +1359,9 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                     outBuff.size = outBuff.pos + adjustedDstSize;
                     DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize);
                     remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
-                    CHECK(ZSTD_isError(remainingToFlush), "ZSTD_compress_generic w/ ZSTD_e_end error : %s", ZSTD_getErrorName(remainingToFlush));
+                    CHECK(ZSTD_isError(remainingToFlush),
+                        "ZSTD_compress_generic w/ ZSTD_e_end error : %s",
+                        ZSTD_getErrorName(remainingToFlush) );
             }   }
             crcOrig = XXH64_digest(&xxhState);
             cSize = outBuff.pos;
@@ -1471,7 +1480,7 @@ int main(int argc, const char** argv)
     e_api selected_api = simple_api;
     const char* const programName = argv[0];
     ZSTD_customMem const customMem = { allocFunction, freeFunction, NULL };
-    ZSTD_customMem const customNULL = { NULL, NULL, NULL };
+    ZSTD_customMem const customNULL = ZSTD_defaultCMem;
 
     /* Check command line */
     for(argNb=1; argNb
Date: Mon, 19 Jun 2017 11:53:42 -0700
Subject: [PATCH 083/109] minor declaration statement warning fix

---
 tests/zstreamtest.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 57993fcbb..95b3add0b 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1305,7 +1305,6 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
 
                 if (dict && dictSize) {
                     /* test that compression parameters are correctly rejected after setting a dictionary */
-                    DISPLAYLEVEL(5, "setting windowLog while dict!=NULL \n");
                     size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1) ;
                     CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed");
                 }

From c7dcf0f37937b5af99730072ec048f94506b5648 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 19 Jun 2017 12:03:25 -0700
Subject: [PATCH 084/109] updated manual

---
 doc/zstd_manual.html | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index d108acaa4..907366032 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -437,8 +437,9 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
          If it needs more memory than available, it will simply error out.
   Note 2 : there is no corresponding "free" function.
            Since workspace was allocated externally, it must be freed externally too.
-  Limitation : currently not compatible with internal CDict creation, such as
-               ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
+  Limitation 1 : currently not compatible with internal CDict creation, such as
+                 ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
+  Limitation 2 : currently not compatible with multi-threading
  
 


 
@@ -496,8 +497,8 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
 


 
 
ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
-
   optimize params for a given `srcSize` and `dictSize`.
-   both values are optional, select `0` if unknown. 
+
  optimize params for a given `srcSize` and `dictSize`.
+  both values are optional, select `0` if unknown. 
 


 
 
size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
@@ -611,7 +612,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
            this value is overriden by srcSize instead. 
 


 
-
size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);   /* Not ready yet ! */
+
size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 
  Create an internal CDict from dict buffer.
   Decompression will have to use same buffer.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).

From bf99150be30b19009592414d763e82570ff53df3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 19 Jun 2017 12:56:25 -0700
Subject: [PATCH 085/109] update new api presentation in zstd.h and manual

---
 doc/zstd_manual.html         | 395 +++++++++++++++--------------
 lib/compress/zstd_compress.c |   6 +-
 lib/zstd.h                   | 465 ++++++++++++++++++-----------------
 3 files changed, 431 insertions(+), 435 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 907366032..cf51daa85 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -516,204 +516,6 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
 
   Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters 
 


 
-
New experimental advanced parameters API


-
typedef enum {
-    /* compression parameters */
-    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
-                              * Default level is 3.
-                              * Special: value 0 means "do not change cLevel". */
-    ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
-                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
-                              * default value : set through compressionLevel.
-                              * Special: value 0 means "do not change windowLog". */
-    ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
-                              * Resulting table size is (1 << (hashLog+2)).
-                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
-                              * Larger tables improve compression ratio of strategies <= dFast,
-                              * and improve speed of strategies > dFast.
-                              * Special: value 0 means "do not change hashLog". */
-    ZSTD_p_chainLog,         /* Size of the full-search table, as a power of 2.
-                              * Resulting table size is (1 << (chainLog+2)).
-                              * Larger tables result in better and slower compression.
-                              * This parameter is useless when using "fast" strategy.
-                              * Special: value 0 means "do not change chainLog". */
-    ZSTD_p_searchLog,        /* Number of search attempts, as a power of 2.
-                              * More attempts result in better and slower compression.
-                              * This parameter is useless when using "fast" and "dFast" strategies.
-                              * Special: value 0 means "do not change searchLog". */
-    ZSTD_p_minMatch,         /* Minimum size of searched matches (note : repCode matches can be smaller).
-                              * Larger values make faster compression and decompression, but decrease ratio.
-                              * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
-                              * Note that currently, for all strategies < btopt, effective minimum is 4.
-                              * Note that currently, for all strategies > fast, effective maximum is 6.
-                              * Special: value 0 means "do not change minMatchLength". */
-    ZSTD_p_targetLength,     /* Only useful for strategies >= btopt.
-                              * Length of Match considered "good enough" to stop search.
-                              * Larger values make compression stronger and slower.
-                              * Special: value 0 means "do not change targetLength". */
-    ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
-                              * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
-                              * The higher the value of selected strategy, the more complex it is,
-                              * resulting in stronger and slower compression.
-                              * Special: value 0 means "do not change strategy". */
-#if 0
-    ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
-                              * Can be set to a more precise value than windowLog.
-                              * Will be transparently reduced to closest possible inferior value
-                              * (see Zstandard compression format) */
-                             /* Not ready yet ! */
-#endif
-
-    /* frame parameters */
-    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
-    ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
-    ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
-
-    /* dictionary parameters */
-    ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
-                              * This avoids duplicating dictionary content.
-                              * But it also requires that dictionary buffer outlives its user (CDict) */
-                             /* Not ready yet ! */
-    ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
-                             /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
-
-    /* multi-threading parameters */
-    ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
-                              * More threads improve speed, but also increase memory usage.
-                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
-                              * Special: value 0 means "do not change nbThreads" */
-    ZSTD_p_jobSize,          /* Size of a compression job. Each compression job is completed in parallel.
-                              * 0 means default, which is dynamically determined based on compression parameters.
-                              * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
-                              * The minimum size is automatically and transparently enforced */
-    ZSTD_p_overlapSizeLog,   /* Size of previous input reloaded at the beginning of each job.
-                              * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-
-    /* advanced parameters - may not remain available after API update */
-    ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
-                              * even when referencing into Dictionary content
-                              * default : 0 when using a CDict, 1 when using a Prefix */
-} ZSTD_cParameter;
-


-
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
-
  Set one compression parameter, selected by enum ZSTD_cParameter.
-  Note : when `value` is an enum, cast it to unsigned for proper type checking.
-  @result : 0, or an error code (which can be tested with ZSTD_isError()). 
-


-
-
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
-
  Total input data size to be compressed into a single frame.
-  This value will be controlled at the end, and result in error if not respected.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
-  Note 1 : 0 means zero, empty.
-           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
-           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for all new compression jobs.
-  Note 2 : If all data is provided and consumed in a single round,
-           this value is overriden by srcSize instead. 
-


-
-
size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
-
  Create an internal CDict from dict buffer.
-  Decompression will have to use same buffer.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
-  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix,
-            meaning "return to no-dictionary mode".
-  Note 1 : Dictionary content will be copied internally,
-           except if ZSTD_p_refDictContent is set.
-  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
-           For this reason, compression parameters cannot be changed anymore after loading a prefix.
-           It's also a CPU-heavy operation, with non-negligible impact on latency.
-  Note 3 : Dictionary will be used for all future compression jobs.
-           To return to "no-dictionary" situation, load a NULL dictionary 
-


-
-
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
-
  Reference a prefix (content-only dictionary) to bootstrap next compression job.
-  Decompression will have to use same prefix.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
-  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
-  Note 1 : Prefix buffer is referenced. It must outlive compression job.
-  Note 3 : Prefix is only used once. Tables are discarded at end of compression job.
-           If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
-  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
-           For this reason, compression parameters cannot be changed anymore after loading a prefix.
-           It's also a CPU-heavy operation, with non-negligible impact on latency. 
-


-
-
ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
-size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
-size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
-
  Create a CDict object which is still mutable after creation.
-  It's the only one case allowing usage of ZSTD_CDict_setParameter().
-  Once all compression parameters are selected,
-  it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
-  Dictionary content will be copied internally (except if ZSTD_p_refDictContent is set).
-  After loading the dictionary, no more change is possible.
-  The only remaining operation is to free CDict object.
-  Note : An unfinished CDict behaves the same as a NULL CDict if referenced into a CCtx.
- 
-


-
-
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
-
  Add a prepared dictionary to cctx, to be used for next compression jobs.
-  Note that compression parameters will be enforced from within CDict.
-  Currently, they supercede any compression parameter previously set within CCtx.
-  The dictionary will remain valid for future compression jobs using same cctx.
- @result : 0, or an error code (which can be tested with ZSTD_isError()).
-  Special : adding a NULL CDict means "return to no-dictionary mode".
-  Note 1 : Currently, only one dictionary can be managed.
-           Adding a new dictionary effectively "discards" any previous one.
-  Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
- 
-


-
-
typedef enum {
-    ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
-    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
-    ZSTD_e_end         /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
-} ZSTD_EndDirective;
-


-
size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
-                              ZSTD_outBuffer* output,
-                              ZSTD_inBuffer* input,
-                              ZSTD_EndDirective endOp);
-
  Behave about the same as ZSTD_compressStream. To note :
-  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_setCCtxParameter()
-  - Compression parameters cannot be changed once compression is started.
-  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
-  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
-  - @return provides the amount of data ready to flush within internal buffers
-            or an error code, which can be tested using ZSTD_isError().
-            if @return != 0, flush is not fully completed, and must be called again to empty internal buffers.
-  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
-            only ZSTD_e_end and ZSTD_e_flush operations are allowed.
-            It is necessary to fully flush internal buffers
-            before changing compression parameters or start a new compression job.
- 
-


-
-
void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
-
  Return a CCtx to clean state.
-  Useful after an error, or to interrupt an ongoing compression job and start a new one.
-  Any internal data not yet flushed is cancelled.
-  Dictionary (if any) is dropped, next compression starts with srcSize==unknown by default.
-  It's possible to modify compression parameters after a reset.
- 
-


-
-
size_t ZSTD_compress_generic_simpleArgs (
-                ZSTD_CCtx* cctx,
-                void* dst, size_t dstCapacity, size_t* dstPos,
-          const void* src, size_t srcSize, size_t* srcPos,
-                ZSTD_EndDirective endOp);
-
  Same as ZSTD_compress_generic(),
-  but using only simple integral types as arguments.
-  Argument list is less expressive than ZSTD_{in,out}Buffer,
-  but can be helpful for binders to dynamic languages
-  which have troubles handling structures containing memory pointers.
- 
-


-
 
Advanced decompression functions

 
 
unsigned ZSTD_isFrame(const void* buffer, size_t size);
@@ -931,6 +733,203 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
 


 
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 


+
New advanced API (experimental, and compression only)


+
typedef enum {
+    /* compression parameters */
+    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
+                              * Default level is ZSTD_CLEVEL_DEFAULT==3.
+                              * Special: value 0 means "do not change cLevel". */
+    ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
+                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+                              * default value : set through compressionLevel.
+                              * Special: value 0 means "do not change windowLog". */
+    ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
+                              * Resulting table size is (1 << (hashLog+2)).
+                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+                              * Larger tables improve compression ratio of strategies <= dFast,
+                              * and improve speed of strategies > dFast.
+                              * Special: value 0 means "do not change hashLog". */
+    ZSTD_p_chainLog,         /* Size of the full-search table, as a power of 2.
+                              * Resulting table size is (1 << (chainLog+2)).
+                              * Larger tables result in better and slower compression.
+                              * This parameter is useless when using "fast" strategy.
+                              * Special: value 0 means "do not change chainLog". */
+    ZSTD_p_searchLog,        /* Number of search attempts, as a power of 2.
+                              * More attempts result in better and slower compression.
+                              * This parameter is useless when using "fast" and "dFast" strategies.
+                              * Special: value 0 means "do not change searchLog". */
+    ZSTD_p_minMatch,         /* Minimum size of searched matches (note : repCode matches can be smaller).
+                              * Larger values make faster compression and decompression, but decrease ratio.
+                              * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
+                              * Note that currently, for all strategies < btopt, effective minimum is 4.
+                              * Note that currently, for all strategies > fast, effective maximum is 6.
+                              * Special: value 0 means "do not change minMatchLength". */
+    ZSTD_p_targetLength,     /* Only useful for strategies >= btopt.
+                              * Length of Match considered "good enough" to stop search.
+                              * Larger values make compression stronger and slower.
+                              * Special: value 0 means "do not change targetLength". */
+    ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
+                              * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
+                              * The higher the value of selected strategy, the more complex it is,
+                              * resulting in stronger and slower compression.
+                              * Special: value 0 means "do not change strategy". */
+#if 0
+    ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
+                              * Can be set to a more precise value than windowLog.
+                              * Will be transparently reduced to closest possible inferior value
+                              * (see Zstandard compression format) */
+                             /* Not ready yet ! */
+#endif
+
+    /* frame parameters */
+    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
+    ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
+    ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
+
+    /* dictionary parameters */
+    ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
+                              * This avoids duplicating dictionary content.
+                              * But it also requires that dictionary buffer outlives its user (CDict) */
+                             /* Not ready yet ! */
+    ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
+                             /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
+
+    /* multi-threading parameters */
+    ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
+                              * More threads improve speed, but also increase memory usage.
+                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
+                              * Special: value 0 means "do not change nbThreads" */
+    ZSTD_p_jobSize,          /* Size of a compression job. Each compression job is completed in parallel.
+                              * 0 means default, which is dynamically determined based on compression parameters.
+                              * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
+                              * The minimum size is automatically and transparently enforced */
+    ZSTD_p_overlapSizeLog,   /* Size of previous input reloaded at the beginning of each job.
+                              * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
+
+    /* advanced parameters - may not remain available after API update */
+    ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
+                              * even when referencing into Dictionary content
+                              * default : 0 when using a CDict, 1 when using a Prefix */
+} ZSTD_cParameter;
+


+
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
+
  Set one compression parameter, selected by enum ZSTD_cParameter.
+  Note : when `value` is an enum, cast it to unsigned for proper type checking.
+  @result : 0, or an error code (which can be tested with ZSTD_isError()). 
+


+
+
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
  Total input data size to be compressed as a single frame.
+  This value will be controlled at the end, and result in error if not respected.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Note 1 : 0 means zero, empty.
+           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
+  Note 2 : If all data is provided and consumed in a single round,
+           this value is overriden by srcSize instead. 
+


+
+
size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
  Create an internal CDict from dict buffer.
+  Decompression will have to use same buffer.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix,
+            meaning "return to no-dictionary mode".
+  Note 1 : Dictionary content will be copied internally,
+           except if ZSTD_p_refDictContent is set.
+  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
+           For this reason, compression parameters cannot be changed anymore after loading a prefix.
+           It's also a CPU-heavy operation, with non-negligible impact on latency.
+  Note 3 : Dictionary will be used for all future compression jobs.
+           To return to "no-dictionary" situation, load a NULL dictionary 
+


+
+
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+
  Ref a prepared dictionary, to be used for all next compression jobs.
+  Note that compression parameters are enforced from within CDict,
+  and supercede any compression parameter previously set within CCtx.
+  The dictionary will remain valid for future compression jobs using same CCtx.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Special : adding a NULL CDict means "return to no-dictionary mode".
+  Note 1 : Currently, only one dictionary can be managed.
+           Adding a new dictionary effectively "discards" any previous one.
+  Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
+ 
+


+
+
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
+
  Reference a prefix (content-only dictionary) to bootstrap next compression job.
+  Decompression will have to use same prefix.
+  Prefix is only used once. Tables are discarded at end of compression job.
+  If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
+ @result : 0, or an error code (which can be tested with ZSTD_isError()).
+  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
+  Note 1 : Prefix buffer is referenced. It must outlive compression job.
+  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+           It's a CPU-heavy operation, with non-negligible impact on latency. 
+


+
+
typedef enum {
+    ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
+    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
+    ZSTD_e_end         /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
+} ZSTD_EndDirective;
+


+
size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
+                              ZSTD_outBuffer* output,
+                              ZSTD_inBuffer* input,
+                              ZSTD_EndDirective endOp);
+
  Behave about the same as ZSTD_compressStream. To note :
+  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_setCCtxParameter()
+  - Compression parameters cannot be changed once compression is started.
+  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
+  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
+  - @return provides the minimum amount of data still to flush from internal buffers
+            or an error code, which can be tested using ZSTD_isError().
+            if @return != 0, flush is not fully completed, and must be called again to empty internal buffers.
+  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
+            only ZSTD_e_end and ZSTD_e_flush operations are allowed.
+            It is necessary to fully flush internal buffers
+            before starting a new compression job, or changing compression parameters.
+ 
+


+
+
void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
+
  Return a CCtx to clean state.
+  Useful after an error, or to interrupt an ongoing compression job and start a new one.
+  Any internal data not yet flushed is cancelled.
+  Dictionary (if any) is dropped.
+  It's possible to modify compression parameters after a reset.
+ 
+


+
+
size_t ZSTD_compress_generic_simpleArgs (
+                ZSTD_CCtx* cctx,
+                void* dst, size_t dstCapacity, size_t* dstPos,
+          const void* src, size_t srcSize, size_t* srcPos,
+                ZSTD_EndDirective endOp);
+
  Same as ZSTD_compress_generic(),
+  but using only integral types as arguments.
+  Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
+  but can be helpful for binders from dynamic languages
+  which have troubles handling structures containing memory pointers.
+ 
+


+
+
ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
+size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
+size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
+
  Create a CDict object which is still mutable after creation.
+  It's the only one case allowing usage of ZSTD_CDict_setParameter().
+  Once all compression parameters are selected,
+  it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
+  Dictionary content will be copied internally (except if ZSTD_p_refDictContent is set).
+  After loading the dictionary, no more change is possible.
+  The only remaining operation is to free CDict object.
+  Note : An unfinished CDict behaves the same as a NULL CDict if referenced into a CCtx.
+ 
+


+
 
Block functions
     Block functions produce and decode raw zstd blocks, without frame metadata.
     Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3620e13d8..49d29dce4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -15,6 +15,7 @@
 #  define ZSTD_CLEVEL_DEFAULT 3
 #endif
 
+
 /*-*************************************
 *  Dependencies
 ***************************************/
@@ -4021,11 +4022,6 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
     if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* 0 == default; no negative compressionLevel yet */
     if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
     cp = ZSTD_defaultCParameters[tableID][compressionLevel];
-    if (MEM_32bits()) {   /* auto-correction, for 32-bits mode */
-        if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX;
-        if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
-        if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
-    }
     return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize);   /* no need to ensure initial CParams validity */
 }
 
diff --git a/lib/zstd.h b/lib/zstd.h
index b204b9a7b..a60c91335 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -610,238 +610,6 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                             const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
 
 
-/*===   New experimental advanced parameters API   ===*/
-
-/* notes on API design :
- *   In this proposal, parameters are pushed one by one into an existing CCtx,
- *   and then applied on all following compression jobs.
- *   When no parameter is ever provided, CCtx is created with compression level 3.
- *
- *   Another approach could be to load parameters into an intermediate _opaque_ object.
- *   The object would then be loaded into CCtx (like ZSTD_compress_advanced())
- *   This approach would be compatible with ZSTD_createCDict_advanced().
- *   But it's more cumbersome for CCtx, as it requires to manage an additional object.
- *
- *   Below proposal push parameters directly into CCtx.
- *   It's a bit more complex for CDict, as advanced version now requires 3 steps.
- *   (basic CDict API remains the same).
- *   See API details below
- */
-
-/* notes on naming convention :
- *   Initially, the API favored names like ZSTD_setCCtxParameter() .
- *   In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
- *   The main idea is that it identifies more clearly the target object type.
- *   It feels clearer when considering other potential variants :
- *   ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
- *   ZSTD_DCtx_setParameter()  (rather than ZSTD_setDCtxParameter() )
- *   The left variant looks clearer.
- */
-
-typedef enum {
-    /* compression parameters */
-    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
-                              * Default level is 3.
-                              * Special: value 0 means "do not change cLevel". */
-    ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
-                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
-                              * default value : set through compressionLevel.
-                              * Special: value 0 means "do not change windowLog". */
-    ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
-                              * Resulting table size is (1 << (hashLog+2)).
-                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
-                              * Larger tables improve compression ratio of strategies <= dFast,
-                              * and improve speed of strategies > dFast.
-                              * Special: value 0 means "do not change hashLog". */
-    ZSTD_p_chainLog,         /* Size of the full-search table, as a power of 2.
-                              * Resulting table size is (1 << (chainLog+2)).
-                              * Larger tables result in better and slower compression.
-                              * This parameter is useless when using "fast" strategy.
-                              * Special: value 0 means "do not change chainLog". */
-    ZSTD_p_searchLog,        /* Number of search attempts, as a power of 2.
-                              * More attempts result in better and slower compression.
-                              * This parameter is useless when using "fast" and "dFast" strategies.
-                              * Special: value 0 means "do not change searchLog". */
-    ZSTD_p_minMatch,         /* Minimum size of searched matches (note : repCode matches can be smaller).
-                              * Larger values make faster compression and decompression, but decrease ratio.
-                              * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
-                              * Note that currently, for all strategies < btopt, effective minimum is 4.
-                              * Note that currently, for all strategies > fast, effective maximum is 6.
-                              * Special: value 0 means "do not change minMatchLength". */
-    ZSTD_p_targetLength,     /* Only useful for strategies >= btopt.
-                              * Length of Match considered "good enough" to stop search.
-                              * Larger values make compression stronger and slower.
-                              * Special: value 0 means "do not change targetLength". */
-    ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
-                              * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
-                              * The higher the value of selected strategy, the more complex it is,
-                              * resulting in stronger and slower compression.
-                              * Special: value 0 means "do not change strategy". */
-#if 0
-    ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
-                              * Can be set to a more precise value than windowLog.
-                              * Will be transparently reduced to closest possible inferior value
-                              * (see Zstandard compression format) */
-                             /* Not ready yet ! */
-#endif
-
-    /* frame parameters */
-    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
-    ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
-    ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
-
-    /* dictionary parameters */
-    ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
-                              * This avoids duplicating dictionary content.
-                              * But it also requires that dictionary buffer outlives its user (CDict) */
-                             /* Not ready yet ! */
-    ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
-                             /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
-
-    /* multi-threading parameters */
-    ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
-                              * More threads improve speed, but also increase memory usage.
-                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
-                              * Special: value 0 means "do not change nbThreads" */
-    ZSTD_p_jobSize,          /* Size of a compression job. Each compression job is completed in parallel.
-                              * 0 means default, which is dynamically determined based on compression parameters.
-                              * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
-                              * The minimum size is automatically and transparently enforced */
-    ZSTD_p_overlapSizeLog,   /* Size of previous input reloaded at the beginning of each job.
-                              * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
-
-    /* advanced parameters - may not remain available after API update */
-    ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
-                              * even when referencing into Dictionary content
-                              * default : 0 when using a CDict, 1 when using a Prefix */
-} ZSTD_cParameter;
-
-
-/*! ZSTD_CCtx_setParameter() :
- *  Set one compression parameter, selected by enum ZSTD_cParameter.
- *  Note : when `value` is an enum, cast it to unsigned for proper type checking.
- *  @result : 0, or an error code (which can be tested with ZSTD_isError()). */
-ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
-
-/*! ZSTD_CCtx_setPledgedSrcSize() :
- *  Total input data size to be compressed into a single frame.
- *  This value will be controlled at the end, and result in error if not respected.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Note 1 : 0 means zero, empty.
- *           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
- *           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for all new compression jobs.
- *  Note 2 : If all data is provided and consumed in a single round,
- *           this value is overriden by srcSize instead. */
-ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
-
-/*! ZSTD_CCtx_loadDictionary() :
- *  Create an internal CDict from dict buffer.
- *  Decompression will have to use same buffer.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix,
- *            meaning "return to no-dictionary mode".
- *  Note 1 : Dictionary content will be copied internally,
- *           except if ZSTD_p_refDictContent is set.
- *  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
- *           For this reason, compression parameters cannot be changed anymore after loading a prefix.
- *           It's also a CPU-heavy operation, with non-negligible impact on latency.
- *  Note 3 : Dictionary will be used for all future compression jobs.
- *           To return to "no-dictionary" situation, load a NULL dictionary */
-ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
-
-/*! ZSTD_CCtx_refPrefix() :
- *  Reference a prefix (content-only dictionary) to bootstrap next compression job.
- *  Decompression will have to use same prefix.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
- *  Note 1 : Prefix buffer is referenced. It must outlive compression job.
- *  Note 3 : Prefix is only used once. Tables are discarded at end of compression job.
- *           If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
- *  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
- *           For this reason, compression parameters cannot be changed anymore after loading a prefix.
- *           It's also a CPU-heavy operation, with non-negligible impact on latency. */
-ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
-
-
-/*! ZSTD_CDict_createEmpty() :
- *  Create a CDict object which is still mutable after creation.
- *  It's the only one case allowing usage of ZSTD_CDict_setParameter().
- *  Once all compression parameters are selected,
- *  it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
- *  Dictionary content will be copied internally (except if ZSTD_p_refDictContent is set).
- *  After loading the dictionary, no more change is possible.
- *  The only remaining operation is to free CDict object.
- *  Note : An unfinished CDict behaves the same as a NULL CDict if referenced into a CCtx.
- */
-ZSTDLIB_API ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
-ZSTDLIB_API size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
-ZSTDLIB_API size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
-
-/*! ZSTD_CCtx_refCDict() :
- *  Add a prepared dictionary to cctx, to be used for next compression jobs.
- *  Note that compression parameters will be enforced from within CDict.
- *  Currently, they supercede any compression parameter previously set within CCtx.
- *  The dictionary will remain valid for future compression jobs using same cctx.
- * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Special : adding a NULL CDict means "return to no-dictionary mode".
- *  Note 1 : Currently, only one dictionary can be managed.
- *           Adding a new dictionary effectively "discards" any previous one.
- *  Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
- */
-ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
-
-
-
-typedef enum {
-    ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
-    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
-    ZSTD_e_end         /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
-} ZSTD_EndDirective;
-
-/*! ZSTD_compressStream_generic() :
- *  Behave about the same as ZSTD_compressStream. To note :
- *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_setCCtxParameter()
- *  - Compression parameters cannot be changed once compression is started.
- *  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
- *  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
- *  - @return provides the amount of data ready to flush within internal buffers
- *            or an error code, which can be tested using ZSTD_isError().
- *            if @return != 0, flush is not fully completed, and must be called again to empty internal buffers.
- *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
- *            only ZSTD_e_end and ZSTD_e_flush operations are allowed.
- *            It is necessary to fully flush internal buffers
- *            before changing compression parameters or start a new compression job.
- */
-ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
-                                          ZSTD_outBuffer* output,
-                                          ZSTD_inBuffer* input,
-                                          ZSTD_EndDirective endOp);
-
-/*! ZSTD_CCtx_reset() :
- *  Return a CCtx to clean state.
- *  Useful after an error, or to interrupt an ongoing compression job and start a new one.
- *  Any internal data not yet flushed is cancelled.
- *  Dictionary (if any) is dropped, next compression starts with srcSize==unknown by default.
- *  It's possible to modify compression parameters after a reset.
- */
-ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
-
-
-/*! ZSTD_compress_generic_simpleArgs() :
- *  Same as ZSTD_compress_generic(),
- *  but using only simple integral types as arguments.
- *  Argument list is less expressive than ZSTD_{in,out}Buffer,
- *  but can be helpful for binders to dynamic languages
- *  which have troubles handling structures containing memory pointers.
- */
-size_t ZSTD_compress_generic_simpleArgs (
-                            ZSTD_CCtx* cctx,
-                            void* dst, size_t dstCapacity, size_t* dstPos,
-                      const void* src, size_t srcSize, size_t* srcPos,
-                            ZSTD_EndDirective endOp);
-
-
-
 /*--- Advanced decompression functions ---*/
 
 /*! ZSTD_isFrame() :
@@ -1081,6 +849,239 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
 typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
 
+
+
+/*===   New advanced API (experimental, and compression only)  ===*/
+
+/* notes on API design :
+ *   In this proposal, parameters are pushed one by one into an existing CCtx,
+ *   and then applied on all subsequent compression jobs.
+ *   When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT.
+ *
+ *   This API is intended to replace all others experimental API.
+ *   It can basically do all other use cases, and even new ones.
+ *   It stands a good chance to become "stable",
+ *   after a reasonable testing period.
+ */
+
+/* note on naming convention :
+ *   Initially, the API favored names like ZSTD_setCCtxParameter() .
+ *   In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
+ *   The main idea is that it identifies more clearly the target object type.
+ *   It feels clearer when considering other potential variants :
+ *   ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
+ *   ZSTD_DCtx_setParameter()  (rather than ZSTD_setDCtxParameter() )
+ *   The left variant feels clearer.
+ */
+
+typedef enum {
+    /* compression parameters */
+    ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
+                              * Default level is ZSTD_CLEVEL_DEFAULT==3.
+                              * Special: value 0 means "do not change cLevel". */
+    ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
+                              * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+                              * default value : set through compressionLevel.
+                              * Special: value 0 means "do not change windowLog". */
+    ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
+                              * Resulting table size is (1 << (hashLog+2)).
+                              * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+                              * Larger tables improve compression ratio of strategies <= dFast,
+                              * and improve speed of strategies > dFast.
+                              * Special: value 0 means "do not change hashLog". */
+    ZSTD_p_chainLog,         /* Size of the full-search table, as a power of 2.
+                              * Resulting table size is (1 << (chainLog+2)).
+                              * Larger tables result in better and slower compression.
+                              * This parameter is useless when using "fast" strategy.
+                              * Special: value 0 means "do not change chainLog". */
+    ZSTD_p_searchLog,        /* Number of search attempts, as a power of 2.
+                              * More attempts result in better and slower compression.
+                              * This parameter is useless when using "fast" and "dFast" strategies.
+                              * Special: value 0 means "do not change searchLog". */
+    ZSTD_p_minMatch,         /* Minimum size of searched matches (note : repCode matches can be smaller).
+                              * Larger values make faster compression and decompression, but decrease ratio.
+                              * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
+                              * Note that currently, for all strategies < btopt, effective minimum is 4.
+                              * Note that currently, for all strategies > fast, effective maximum is 6.
+                              * Special: value 0 means "do not change minMatchLength". */
+    ZSTD_p_targetLength,     /* Only useful for strategies >= btopt.
+                              * Length of Match considered "good enough" to stop search.
+                              * Larger values make compression stronger and slower.
+                              * Special: value 0 means "do not change targetLength". */
+    ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition.
+                              * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
+                              * The higher the value of selected strategy, the more complex it is,
+                              * resulting in stronger and slower compression.
+                              * Special: value 0 means "do not change strategy". */
+#if 0
+    ZSTD_p_windowSize,       /* Maximum allowed back-reference distance.
+                              * Can be set to a more precise value than windowLog.
+                              * Will be transparently reduced to closest possible inferior value
+                              * (see Zstandard compression format) */
+                             /* Not ready yet ! */
+#endif
+
+    /* frame parameters */
+    ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
+    ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
+    ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
+
+    /* dictionary parameters */
+    ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
+                              * This avoids duplicating dictionary content.
+                              * But it also requires that dictionary buffer outlives its user (CDict) */
+                             /* Not ready yet ! */
+    ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
+                             /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
+
+    /* multi-threading parameters */
+    ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
+                              * More threads improve speed, but also increase memory usage.
+                              * Can only receive a value > 1 if ZSTD_MULTITHREAD is enabled.
+                              * Special: value 0 means "do not change nbThreads" */
+    ZSTD_p_jobSize,          /* Size of a compression job. Each compression job is completed in parallel.
+                              * 0 means default, which is dynamically determined based on compression parameters.
+                              * Job size must be a minimum of overlapSize, or 1 KB, whichever is largest
+                              * The minimum size is automatically and transparently enforced */
+    ZSTD_p_overlapSizeLog,   /* Size of previous input reloaded at the beginning of each job.
+                              * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
+
+    /* advanced parameters - may not remain available after API update */
+    ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
+                              * even when referencing into Dictionary content
+                              * default : 0 when using a CDict, 1 when using a Prefix */
+} ZSTD_cParameter;
+
+
+/*! ZSTD_CCtx_setParameter() :
+ *  Set one compression parameter, selected by enum ZSTD_cParameter.
+ *  Note : when `value` is an enum, cast it to unsigned for proper type checking.
+ *  @result : 0, or an error code (which can be tested with ZSTD_isError()). */
+ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
+
+/*! ZSTD_CCtx_setPledgedSrcSize() :
+ *  Total input data size to be compressed as a single frame.
+ *  This value will be controlled at the end, and result in error if not respected.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Note 1 : 0 means zero, empty.
+ *           In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
+ *           Note that ZSTD_CONTENTSIZE_UNKNOWN is default value for new compression jobs.
+ *  Note 2 : If all data is provided and consumed in a single round,
+ *           this value is overriden by srcSize instead. */
+ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
+
+/*! ZSTD_CCtx_loadDictionary() :
+ *  Create an internal CDict from dict buffer.
+ *  Decompression will have to use same buffer.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix,
+ *            meaning "return to no-dictionary mode".
+ *  Note 1 : Dictionary content will be copied internally,
+ *           except if ZSTD_p_refDictContent is set.
+ *  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
+ *           For this reason, compression parameters cannot be changed anymore after loading a prefix.
+ *           It's also a CPU-heavy operation, with non-negligible impact on latency.
+ *  Note 3 : Dictionary will be used for all future compression jobs.
+ *           To return to "no-dictionary" situation, load a NULL dictionary */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+
+/*! ZSTD_CCtx_refCDict() :
+ *  Ref a prepared dictionary, to be used for all next compression jobs.
+ *  Note that compression parameters are enforced from within CDict,
+ *  and supercede any compression parameter previously set within CCtx.
+ *  The dictionary will remain valid for future compression jobs using same CCtx.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : adding a NULL CDict means "return to no-dictionary mode".
+ *  Note 1 : Currently, only one dictionary can be managed.
+ *           Adding a new dictionary effectively "discards" any previous one.
+ *  Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
+ */
+ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+
+/*! ZSTD_CCtx_refPrefix() :
+ *  Reference a prefix (content-only dictionary) to bootstrap next compression job.
+ *  Decompression will have to use same prefix.
+ *  Prefix is only used once. Tables are discarded at end of compression job.
+ *  If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
+ * @result : 0, or an error code (which can be tested with ZSTD_isError()).
+ *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
+ *  Note 1 : Prefix buffer is referenced. It must outlive compression job.
+ *  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
+ *           It's a CPU-heavy operation, with non-negligible impact on latency. */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
+
+
+
+typedef enum {
+    ZSTD_e_continue=0, /* collect more data, encoder transparently decides when to output result, for optimal conditions */
+    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
+    ZSTD_e_end         /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
+} ZSTD_EndDirective;
+
+/*! ZSTD_compressStream_generic() :
+ *  Behave about the same as ZSTD_compressStream. To note :
+ *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_setCCtxParameter()
+ *  - Compression parameters cannot be changed once compression is started.
+ *  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
+ *  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
+ *  - @return provides the minimum amount of data still to flush from internal buffers
+ *            or an error code, which can be tested using ZSTD_isError().
+ *            if @return != 0, flush is not fully completed, and must be called again to empty internal buffers.
+ *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
+ *            only ZSTD_e_end and ZSTD_e_flush operations are allowed.
+ *            It is necessary to fully flush internal buffers
+ *            before starting a new compression job, or changing compression parameters.
+ */
+ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
+                                          ZSTD_outBuffer* output,
+                                          ZSTD_inBuffer* input,
+                                          ZSTD_EndDirective endOp);
+
+/*! ZSTD_CCtx_reset() :
+ *  Return a CCtx to clean state.
+ *  Useful after an error, or to interrupt an ongoing compression job and start a new one.
+ *  Any internal data not yet flushed is cancelled.
+ *  Dictionary (if any) is dropped.
+ *  It's possible to modify compression parameters after a reset.
+ */
+ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);   /* Not ready yet ! */
+
+
+/*! ZSTD_compress_generic_simpleArgs() :
+ *  Same as ZSTD_compress_generic(),
+ *  but using only integral types as arguments.
+ *  Argument list is larger and less expressive than ZSTD_{in,out}Buffer,
+ *  but can be helpful for binders from dynamic languages
+ *  which have troubles handling structures containing memory pointers.
+ */
+size_t ZSTD_compress_generic_simpleArgs (
+                            ZSTD_CCtx* cctx,
+                            void* dst, size_t dstCapacity, size_t* dstPos,
+                      const void* src, size_t srcSize, size_t* srcPos,
+                            ZSTD_EndDirective endOp);
+
+
+/* note : this part if completely experimental, not ready yet.
+ * Main idea is : if ZSTD_parameters is not a good fit for stable API,
+ * there will be a need for something else.
+ * While initialising compression is now correctly covered,
+ * creating an advanced CDict still needs to be solved.
+ * Below is merely a plan of what it could look like */
+/*! ZSTD_CDict_createEmpty() :
+ *  Create a CDict object which is still mutable after creation.
+ *  It's the only one case allowing usage of ZSTD_CDict_setParameter().
+ *  Once all compression parameters are selected,
+ *  it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
+ *  Dictionary content will be copied internally (except if ZSTD_p_refDictContent is set).
+ *  After loading the dictionary, no more change is possible.
+ *  The only remaining operation is to free CDict object.
+ *  Note : An unfinished CDict behaves the same as a NULL CDict if referenced into a CCtx.
+ */
+ZSTDLIB_API ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
+ZSTDLIB_API size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
+ZSTDLIB_API size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
+
+
 /**
     Block functions
 

From fe234bf48b1d6e8b82aff99237bbea7e8ebf7137 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 19 Jun 2017 15:23:19 -0700
Subject: [PATCH 086/109] fix attempts : fullbench for VS2008

---
 build/VS2008/fullbench/fullbench.vcproj | 54 +++++++++++++----------
 lib/zstd.h                              | 58 ++++++++++++-------------
 programs/bench.c                        | 15 +++----
 3 files changed, 67 insertions(+), 60 deletions(-)

diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj
index 0734219b4..582a9ce93 100644
--- a/build/VS2008/fullbench/fullbench.vcproj
+++ b/build/VS2008/fullbench/fullbench.vcproj
@@ -328,10 +328,6 @@
 			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
 			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
 			>
-			
-			
 			
@@ -340,10 +336,6 @@
 				RelativePath="..\..\..\lib\common\error_private.c"
 				>
 			
-			
-			
 			
@@ -352,18 +344,6 @@
 				RelativePath="..\..\..\lib\common\xxhash.c"
 				>
 			
-			
-			
-			
-			
-			
-			
 			
@@ -372,16 +352,44 @@
 				RelativePath="..\..\..\lib\compress\zstd_compress.c"
 				>
 			
+			
+			
+			
+			
+			
+			
 			
 			
+			
+			
+			
+			
+			
+			
 		
 		
+			
+			
 			
@@ -419,11 +427,11 @@
 				>
 			
 			
 			
 			
 			
 			
 			
 			
 			
 		
diff --git a/lib/zstd.h b/lib/zstd.h
index a60c91335..e0c880103 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -47,12 +47,12 @@ extern "C" {
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit memory management)
     - unbounded multiple steps (described as Streaming compression)
-  The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
+  The compression ratio achievable on small data can be highly improved using a dictionary in:
     - a single step (described as Simple dictionary API)
     - a single step, reusing a dictionary (described as Fast dictionary API)
 
   Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
-  These APIs shall never be used with a dynamic library.
+  Advanced experimental APIs shall never be used with a dynamic library.
   They are not "stable", their definition may change in the future. Only static linking is allowed.
 *********************************************************************************************************/
 
@@ -62,13 +62,13 @@ extern "C" {
 #define ZSTD_VERSION_RELEASE  0
 
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
-ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< to be used when checking dll version */
+ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< useful to check dll version */
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
 #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
 #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
-ZSTDLIB_API const char* ZSTD_versionString(void);   /* v1.3.0 */
+ZSTDLIB_API const char* ZSTD_versionString(void);   /* >= v1.3.0 */
 
 
 /***************************************
@@ -85,7 +85,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
 
 /*! ZSTD_decompress() :
  *  `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
- *  `dstCapacity` is an upper bound of originalSize.
+ *  `dstCapacity` is an upper bound of originalSize to regenerate.
  *  If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
  *  @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
  *            or an errorCode if it fails (which can be tested using ZSTD_isError()). */
@@ -163,9 +163,9 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx,
 *  Simple dictionary API
 ***************************/
 /*! ZSTD_compress_usingDict() :
-*   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
-*   Note : This function loads the dictionary, resulting in significant startup delay.
-*   Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ *  Compression using a predefined Dictionary (see dictBuilder/zdict.h).
+ *  Note : This function loads the dictionary, resulting in significant startup delay.
+ *  Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
                                            void* dst, size_t dstCapacity,
                                      const void* src, size_t srcSize,
@@ -173,31 +173,31 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
                                            int compressionLevel);
 
 /*! ZSTD_decompress_usingDict() :
-*   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
-*   Dictionary must be identical to the one used during compression.
-*   Note : This function loads the dictionary, resulting in significant startup delay.
-*   Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
+ *  Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
+ *  Dictionary must be identical to the one used during compression.
+ *  Note : This function loads the dictionary, resulting in significant startup delay.
+ *  Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                              void* dst, size_t dstCapacity,
                                        const void* src, size_t srcSize,
                                        const void* dict,size_t dictSize);
 
 
-/****************************
-*  Fast dictionary API
-****************************/
+/**********************************
+ *  Bulk processing dictionary API
+ *********************************/
 typedef struct ZSTD_CDict_s ZSTD_CDict;
 
 /*! ZSTD_createCDict() :
-*   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
-*   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
-*   ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-*   `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
+ *  When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+ *  ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+ *  ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+ *  `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
                                          int compressionLevel);
 
 /*! ZSTD_freeCDict() :
-*   Function frees memory allocated by ZSTD_createCDict(). */
+ *  Function frees memory allocated by ZSTD_createCDict(). */
 ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
 
 /*! ZSTD_compress_usingCDict() :
@@ -214,17 +214,17 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
 typedef struct ZSTD_DDict_s ZSTD_DDict;
 
 /*! ZSTD_createDDict() :
-*   Create a digested dictionary, ready to start decompression operation without startup delay.
-*   dictBuffer can be released after DDict creation, as its content is copied inside DDict */
+ *  Create a digested dictionary, ready to start decompression operation without startup delay.
+ *  dictBuffer can be released after DDict creation, as its content is copied inside DDict */
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
 
 /*! ZSTD_freeDDict() :
-*   Function frees memory allocated with ZSTD_createDDict() */
+ *  Function frees memory allocated with ZSTD_createDDict() */
 ZSTDLIB_API size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
 
 /*! ZSTD_decompress_usingDDict() :
-*   Decompression using a digested Dictionary.
-*   Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
+ *  Decompression using a digested Dictionary.
+ *  Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
 ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                                               void* dst, size_t dstCapacity,
                                         const void* src, size_t srcSize,
@@ -290,8 +290,8 @@ typedef struct ZSTD_outBuffer_s {
 *
 * *******************************************************************/
 
-typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are effectively same object */
-                                 /* Continue due distinghish them for compatibility with versions <= v1.2.0 */
+typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
+                                 /* But continue to distinguish them for compatibility with versions <= v1.2.0 */
 /*===== ZSTD_CStream management functions =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@@ -329,8 +329,8 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 *            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
 * *******************************************************************************/
 
-//typedef struct ZSTD_DStream_s ZSTD_DStream;
-typedef ZSTD_DCtx ZSTD_DStream;
+typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
+                                 /* But continue to distinguish them for compatibility with versions <= v1.2.0 */
 /*===== ZSTD_DStream management functions =====*/
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
diff --git a/programs/bench.c b/programs/bench.c
index 22b871952..1f1b00594 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -86,14 +86,13 @@ static clock_t g_time = 0;
 #ifndef DEBUG
 #  define DEBUG 0
 #endif
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...)                                             \
-{                                                                         \
-    DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
-    DISPLAYLEVEL(1, "Error %i : ", error);                                \
-    DISPLAYLEVEL(1, __VA_ARGS__);                                         \
-    DISPLAYLEVEL(1, " \n");                                               \
-    exit(error);                                                          \
+#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
+#define EXM_THROW(error, ...)  {                      \
+    DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__);    \
+    DISPLAYLEVEL(1, "Error %i : ", error);            \
+    DISPLAYLEVEL(1, __VA_ARGS__);                     \
+    DISPLAYLEVEL(1, " \n");                           \
+    exit(error);                                      \
 }
 
 

From 695a0a34496f43e2fbd184c247a71bdf9711eb1b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 19 Jun 2017 15:27:30 -0700
Subject: [PATCH 087/109] fixed IA64 compilation error, by @mcmilk

---
 lib/decompress/zstd_decompress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index bb3eed399..aeaa40c1f 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -53,7 +53,7 @@
 #  include "zstd_legacy.h"
 #endif
 
-#if defined(_MSC_VER)
+#if defined(_MSC_VER) && !defined(_M_IA64)  /* _mm_prefetch() is not defined for ia64 */
 #  include    /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
 #  define ZSTD_PREFETCH(ptr)   _mm_prefetch((const char*)ptr, _MM_HINT_T0)
 #elif defined(__GNUC__)

From 008d44ad66dc09cbbd4cc2ad782559731aeee8d2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 19 Jun 2017 17:45:43 -0700
Subject: [PATCH 088/109] fix attempt : fullbench VS2008 project

---
 build/VS2008/fullbench/fullbench.vcproj | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj
index 582a9ce93..942934561 100644
--- a/build/VS2008/fullbench/fullbench.vcproj
+++ b/build/VS2008/fullbench/fullbench.vcproj
@@ -348,6 +348,14 @@
 				RelativePath="..\..\..\lib\common\zstd_common.c"
 				>
 			
+			
+			
+			
+			
 			

From c08e649e95deb7fdee73bc13e71ee067419d22bf Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 19 Jun 2017 18:25:35 -0700
Subject: [PATCH 089/109] first implementation of bench.c with new API
 ZSTD_compress_generic()

Doesn't speed optimize this buffer-to-buffer scenario yet.
Still internally defers to streaming implementation.

Also : fixed a long standing bug in ZSTDMT streaming API.
---
 doc/zstd_manual.html           | 50 ++++++++++++------------
 lib/compress/zstd_compress.c   |  5 ++-
 lib/compress/zstdmt_compress.c | 70 ++++++++++++++++++++--------------
 programs/bench.c               | 51 +++++++++++++++++++------
 4 files changed, 110 insertions(+), 66 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index cf51daa85..88e890793 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -13,7 +13,7 @@
 
  • Simple API
    • 
 
  • Explicit memory management
    • 
 
  • Simple dictionary API
    • 
-
  • Fast dictionary API
    • 
+
  • Bulk processing dictionary API
    • 
 
  • Streaming
    • 
 
  • Streaming compression - HowTo
    • 
 
  • Streaming decompression - HowTo
    • 
@@ -40,18 +40,18 @@
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit memory management)
     - unbounded multiple steps (described as Streaming compression)
-  The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
+  The compression ratio achievable on small data can be highly improved using a dictionary in:
     - a single step (described as Simple dictionary API)
     - a single step, reusing a dictionary (described as Fast dictionary API)
 
   Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
-  These APIs shall never be used with a dynamic library.
+  Advanced experimental APIs shall never be used with a dynamic library.
   They are not "stable", their definition may change in the future. Only static linking is allowed.
 
    
 
 
    Version
    
 
-
    unsigned ZSTD_versionNumber(void);   /**< to be used when checking dll version */
+
    unsigned ZSTD_versionNumber(void);   /**< useful to check dll version */
 

    
 
    Simple API
    
 
@@ -67,7 +67,7 @@
 
    size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                   const void* src, size_t compressedSize);
 
      `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
-  `dstCapacity` is an upper bound of originalSize.
+  `dstCapacity` is an upper bound of originalSize to regenerate.
   If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
   @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
             or an errorCode if it fails (which can be tested using ZSTD_isError()). 
@@ -140,33 +140,33 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                          const void* src, size_t srcSize,
                          const void* dict,size_t dictSize,
                                int compressionLevel);
-
       Compression using a predefined Dictionary (see dictBuilder/zdict.h).
-   Note : This function loads the dictionary, resulting in significant startup delay.
-   Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
+
      Compression using a predefined Dictionary (see dictBuilder/zdict.h).
+  Note : This function loads the dictionary, resulting in significant startup delay.
+  Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
 

    
 
 
    size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                  void* dst, size_t dstCapacity,
                            const void* src, size_t srcSize,
                            const void* dict,size_t dictSize);
-
       Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
-   Dictionary must be identical to the one used during compression.
-   Note : This function loads the dictionary, resulting in significant startup delay.
-   Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
+
      Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
+  Dictionary must be identical to the one used during compression.
+  Note : This function loads the dictionary, resulting in significant startup delay.
+  Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
 

    
 
-
    Fast dictionary API
    
+
    Bulk processing dictionary API
    
 
 
    ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
                              int compressionLevel);
-
       When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
-   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
-   ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-   `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict 
+
      When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+  ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+  ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
+  `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict 
 

    
 
 
    size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
-
       Function frees memory allocated by ZSTD_createCDict(). 
+
      Function frees memory allocated by ZSTD_createCDict(). 
 

    
 
 
    size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
@@ -180,20 +180,20 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 

    
 
 
    ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
-
       Create a digested dictionary, ready to start decompression operation without startup delay.
-   dictBuffer can be released after DDict creation, as its content is copied inside DDict 
+
      Create a digested dictionary, ready to start decompression operation without startup delay.
+  dictBuffer can be released after DDict creation, as its content is copied inside DDict 
 

    
 
 
    size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
-
       Function frees memory allocated with ZSTD_createDDict() 
+
      Function frees memory allocated with ZSTD_createDDict() 
 

    
 
 
    size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                                   void* dst, size_t dstCapacity,
                             const void* src, size_t srcSize,
                             const ZSTD_DDict* ddict);
-
       Decompression using a digested Dictionary.
-   Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. 
+
      Decompression using a digested Dictionary.
+  Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. 
 

    
 
 
    Streaming
    
@@ -250,7 +250,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
  
 
    
 
-
    typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are effectively same object */
+
    typedef ZSTD_CCtx ZSTD_CStream;  /**< CCtx and CStream are now effectively same object (>= v1.3.0) */
 

    
 
    ZSTD_CStream management functions
    ZSTD_CStream* ZSTD_createCStream(void);
 size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
@@ -285,6 +285,8 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
  
 
    
 
+
    typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
+

    
 
    ZSTD_DStream management functions
    ZSTD_DStream* ZSTD_createDStream(void);
 size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 

    
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 49d29dce4..f5ac75a95 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -571,7 +571,7 @@ static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
 }
 
 /*! ZSTD_continueCCtx() :
-    reuse CCtx without reset (note : requires no dictionary) */
+ *  reuse CCtx without reset (note : requires no dictionary) */
 static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 pledgedSrcSize)
 {
     U32 const end = (U32)(cctx->nextSrc - cctx->base);
@@ -3831,6 +3831,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
 
 #ifdef ZSTD_MULTITHREAD
         if (cctx->nbThreads > 1) {
+            DEBUGLOG(4, "call ZSTDMT_initCStream_internal");
             CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, NULL, 0, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
             cctx->streamStage = zcss_load;
         } else
@@ -3842,7 +3843,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
 #ifdef ZSTD_MULTITHREAD
     if (cctx->nbThreads > 1) {
         size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
-        DEBUGLOG(5, "ZSTDMT result : %u", (U32)flushMin);
+        DEBUGLOG(4, "ZSTDMT_compressStream_generic : %u", (U32)flushMin);
         if ( ZSTD_isError(flushMin)
           || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
             ZSTD_startNewCompression(cctx);
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 42e31ca18..895d362fb 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -190,7 +190,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
     cctxPool->availCCtx = 1;   /* at least one cctx for single-thread mode */
     cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem);
     if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
-    DEBUGLOG(3, "cctxPool created, with %u threads", nbThreads);
+    DEBUGLOG(4, "cctxPool created, with %u threads", nbThreads);
     return cctxPool;
 }
 
@@ -261,11 +261,11 @@ void ZSTDMT_compressChunk(void* jobDescription)
     ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
     const void* const src = (const char*)job->srcStart + job->dictSize;
     buffer_t const dstBuff = job->dstBuff;
-    DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
+    DEBUGLOG(4, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
                  job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
     if (job->cdict) {  /* should only happen for first segment */
         size_t const initError = ZSTD_compressBegin_usingCDict_advanced(job->cctx, job->cdict, job->params.fParams, job->fullFrameSize);
-        DEBUGLOG(3, "using CDict");
+        DEBUGLOG(5, "using CDict");
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
     } else {  /* srcStart points at reloaded section */
         if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0;  /* ensure no srcSize control */
@@ -280,12 +280,12 @@ void ZSTDMT_compressChunk(void* jobDescription)
         ZSTD_invalidateRepCodes(job->cctx);
     }
 
-    DEBUGLOG(4, "Compressing : ");
+    DEBUGLOG(5, "Compressing : ");
     DEBUG_PRINTHEX(4, job->srcStart, 12);
     job->cSize = (job->lastChunk) ?
                  ZSTD_compressEnd     (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
                  ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
-    DEBUGLOG(3, "compressed %u bytes into %u bytes   (first:%u) (last:%u)",
+    DEBUGLOG(4, "compressed %u bytes into %u bytes   (first:%u) (last:%u)",
                 (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
     DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize));
 
@@ -426,7 +426,7 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter,
         mtctx->sectionSize = value;
         return 0;
     case ZSTDMT_p_overlapSectionLog :
-        DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
+        DEBUGLOG(5, "ZSTDMT_p_overlapSectionLog : %u", value);
         mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value;
         return 0;
     default :
@@ -457,8 +457,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
     unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize));  /* presumes avgChunkSize >= 256 KB, which should be the case */
     size_t frameStartPos = 0, dstBufferPos = 0;
 
-    DEBUGLOG(3, "windowLog : %2u => chunkTargetSize : %u bytes  ", params.cParams.windowLog, (U32)chunkTargetSize);
-    DEBUGLOG(3, "nbChunks  : %2u   (chunkSize : %u bytes)   ", nbChunks, (U32)avgChunkSize);
+    DEBUGLOG(4, "windowLog : %2u => chunkTargetSize : %u bytes  ", params.cParams.windowLog, (U32)chunkTargetSize);
+    DEBUGLOG(4, "nbChunks  : %2u   (chunkSize : %u bytes)   ", nbChunks, (U32)avgChunkSize);
     params.fParams.contentSizeFlag = 1;
 
     if (nbChunks==1) {   /* fallback to single-thread mode */
@@ -495,8 +495,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
             mtctx->jobs[u].jobCompleted_mutex = &mtctx->jobCompleted_mutex;
             mtctx->jobs[u].jobCompleted_cond = &mtctx->jobCompleted_cond;
 
-            DEBUGLOG(3, "posting job %u   (%u bytes)", u, (U32)chunkSize);
-            DEBUG_PRINTHEX(3, mtctx->jobs[u].srcStart, 12);
+            DEBUGLOG(4, "posting job %u   (%u bytes)", u, (U32)chunkSize);
+            DEBUG_PRINTHEX(5, mtctx->jobs[u].srcStart, 12);
             POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]);
 
             frameStartPos += chunkSize;
@@ -508,14 +508,14 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
     {   unsigned chunkID;
         size_t error = 0, dstPos = 0;
         for (chunkID=0; chunkIDjobCompleted_mutex);
             while (mtctx->jobs[chunkID].jobCompleted==0) {
-                DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", chunkID);
+                DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", chunkID);
                 pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
             }
             pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
-            DEBUGLOG(3, "ready to write chunk %u ", chunkID);
+            DEBUGLOG(5, "ready to write chunk %u ", chunkID);
 
             ZSTDMT_releaseCCtx(mtctx->cctxPool, mtctx->jobs[chunkID].cctx);
             mtctx->jobs[chunkID].cctx = NULL;
@@ -533,7 +533,7 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
                 dstPos += cSize ;
             }
         }
-        if (!error) DEBUGLOG(3, "compressed size : %u  ", (U32)dstPos);
+        if (!error) DEBUGLOG(4, "compressed size : %u  ", (U32)dstPos);
         return error ? error : dstPos;
     }
 
@@ -550,7 +550,7 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
         unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
         PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
         while (zcs->jobs[jobID].jobCompleted==0) {
-            DEBUGLOG(4, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID);   /* we want to block when waiting for data to flush */
+            DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID);   /* we want to block when waiting for data to flush */
             pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
         }
         pthread_mutex_unlock(&zcs->jobCompleted_mutex);
@@ -593,12 +593,12 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     }
 
     zcs->targetDictSize = (zcs->overlapRLog>=9) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog);
-    DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog);
-    DEBUGLOG(3, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
+    DEBUGLOG(5, "overlapRLog : %u ", zcs->overlapRLog);
+    DEBUGLOG(5, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10));
     zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2);
     zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize);
     zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize);
-    DEBUGLOG(3, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
+    DEBUGLOG(5, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10));
     zcs->marginSize = zcs->targetSectionSize >> 2;
     zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize;
     zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
@@ -628,7 +628,8 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
     ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
     if (cdict==NULL) return ERROR(GENERIC);   /* method incompatible with NULL cdict */
     params.fParams = fParams;
-    return ZSTDMT_initCStream_internal(mtctx, NULL, 0, cdict, params, pledgedSrcSize);
+    return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, cdict,
+                                        params, pledgedSrcSize);
 }
 
 
@@ -686,6 +687,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
     /* get a new buffer for next input */
     if (!endFrame) {
         size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize);
+        DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame);
         zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize);
         if (zcs->inBuff.buffer.start == NULL) {   /* not enough memory to allocate next input buffer */
             zcs->jobs[jobID].jobCompleted = 1;
@@ -694,9 +696,9 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
             ZSTDMT_releaseAllJobResources(zcs);
             return ERROR(memory_allocation);
         }
-        DEBUGLOG(5, "inBuff filled to %u", (U32)zcs->inBuff.filled);
+        DEBUGLOG(5, "inBuff currently filled to %u", (U32)zcs->inBuff.filled);
         zcs->inBuff.filled -= srcSize + zcs->dictSize - newDictSize;
-        DEBUGLOG(5, "new job : filled to %u, with %u dict and %u src",
+        DEBUGLOG(5, "new job : inBuff filled to %u, with %u dict and %u src",
                     (U32)zcs->inBuff.filled, (U32)newDictSize,
                     (U32)(zcs->inBuff.filled - newDictSize));
         memmove(zcs->inBuff.buffer.start,
@@ -705,6 +707,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
         DEBUGLOG(5, "new inBuff pre-filled");
         zcs->dictSize = newDictSize;
     } else {   /* if (endFrame==1) */
+        DEBUGLOG(5, "ZSTDMT_createCompressionJob::endFrame = %u", endFrame);
         zcs->inBuff.buffer = g_nullBuffer;
         zcs->inBuff.filled = 0;
         zcs->dictSize = 0;
@@ -714,7 +717,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsi
             zcs->params.fParams.checksumFlag = 0;
     }
 
-    DEBUGLOG(3, "posting job %u : %u bytes  (end:%u) (note : doneJob = %u=>%u)",
+    DEBUGLOG(4, "posting job %u : %u bytes  (end:%u) (note : doneJob = %u=>%u)",
                 zcs->nextJobID,
                 (U32)zcs->jobs[jobID].srcSize,
                 zcs->jobs[jobID].lastChunk,
@@ -757,7 +760,7 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
                 XXH64_update(&zcs->xxhState, (const char*)job.srcStart + job.dictSize, job.srcSize);
                 if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) {  /* write checksum at end of last section */
                     U32 const checksum = (U32)XXH64_digest(&zcs->xxhState);
-                    DEBUGLOG(4, "writing checksum : %08X \n", checksum);
+                    DEBUGLOG(5, "writing checksum : %08X \n", checksum);
                     MEM_writeLE32((char*)job.dstBuff.start + job.cSize, checksum);
                     job.cSize += 4;
                     zcs->jobs[wJobID].cSize += 4;
@@ -792,23 +795,25 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
 size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
 {
     size_t const newJobThreshold = zcs->dictSize + zcs->targetSectionSize + zcs->marginSize;
-    if (zcs->frameEnded)
-        /* current frame being ended. Only flush is allowed. Restart with init */
+    if (zcs->frameEnded) {
+        /* current frame being ended. Only flush is allowed. Or start new job with init */
+        DEBUGLOG(5, "ZSTDMT_compressStream: zcs::frameEnded==1");
         return ERROR(stage_wrong);
+    }
     if (zcs->nbThreads==1) {
         return ZSTD_compressStream(zcs->cctxPool->cctx[0], output, input);
     }
 
     /* fill input buffer */
     {   size_t const toLoad = MIN(input->size - input->pos, zcs->inBuffSize - zcs->inBuff.filled);
-        memcpy((char*)zcs->inBuff.buffer.start + zcs->inBuff.filled, input->src, toLoad);
+        memcpy((char*)zcs->inBuff.buffer.start + zcs->inBuff.filled, (const char*)input->src + input->pos, toLoad);
         input->pos += toLoad;
         zcs->inBuff.filled += toLoad;
     }
 
     if ( (zcs->inBuff.filled >= newJobThreshold)  /* filled enough : let's compress */
         && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {   /* avoid overwriting job round buffer */
-        CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0 /* blockToFlush */) );
+        CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0 /* endFrame */) );
     }
 
     /* check for data to flush */
@@ -824,7 +829,8 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* outp
     size_t const srcSize = zcs->inBuff.filled - zcs->dictSize;
 
     if (srcSize)
-        DEBUGLOG(5, "flushing : %u bytes left to compress", (U32)srcSize);
+        DEBUGLOG(5, "ZSTDMT_flushStream_internal : %u bytes left to compress",
+                    (U32)srcSize);
     if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded))
        && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) {
         DEBUGLOG(5, "create new job with %u bytes to compress", (U32)srcSize);
@@ -842,6 +848,7 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* outp
 
 size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
 {
+    DEBUGLOG(5, "ZSTDMT_flushStream");
     if (zcs->nbThreads==1)
         return ZSTD_flushStream(zcs->cctxPool->cctx[0], output);
     return ZSTDMT_flushStream_internal(zcs, output, 0 /* endFrame */);
@@ -849,6 +856,7 @@ size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
 
 size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output)
 {
+    DEBUGLOG(5, "ZSTDMT_endStream");
     if (zcs->nbThreads==1)
         return ZSTD_endStream(zcs->cctxPool->cctx[0], output);
     return ZSTDMT_flushStream_internal(zcs, output, 1 /* endFrame */);
@@ -859,17 +867,21 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
                                      ZSTD_inBuffer* input,
                                      ZSTD_EndDirective endOp)
 {
+    DEBUGLOG(5, "in: pos:%u / size:%u ; endOp=%u",
+                (U32)input->pos, (U32)input->size, (U32)endOp);
     if (input->pos < input->size)  /* exclude final flushes */
         CHECK_F(ZSTDMT_compressStream(mtctx, output, input));
+    if (input->pos < input->size) endOp = ZSTD_e_continue;
     switch(endOp)
     {
         case ZSTD_e_flush:
             return ZSTDMT_flushStream(mtctx, output);
         case ZSTD_e_end:
+            DEBUGLOG(5, "endOp:%u; calling ZSTDMT_endStream", (U32)endOp);
             return ZSTDMT_endStream(mtctx, output);
         case ZSTD_e_continue:
             return 1;
         default:
-            return ERROR(GENERIC);
+            return ERROR(GENERIC);   /* invalid endDirective */
     }
 }
diff --git a/programs/bench.c b/programs/bench.c
index 1f1b00594..6f09a56de 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -158,7 +158,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                         const ZSTD_compressionParameters* comprParams)
 {
     size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
-    size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
     U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
     blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
     size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);   /* add some room for safety */
@@ -261,42 +260,72 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                 UTIL_getTime(&clockStart);
 
                 if (!cCompleted) {   /* still some time to do compression tests */
-                    ZSTD_customMem const cmem = { NULL, NULL, NULL };
                     U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
                     U32 nbLoops = 0;
+                    ZSTD_CDict* cdict = NULL;
+#ifdef ZSTD_NEWAPI
+                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
+                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
+                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog);
+                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog);
+                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog);
+                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength);
+                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
+                    ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
+                    ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
+#else
+                    size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
                     ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
-                    ZSTD_CDict* cdict;
+                    ZSTD_customMem const cmem = { NULL, NULL, NULL };
                     if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
                     if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
                     if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
                     if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
                     if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
                     if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
-                    if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1);
+                    if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
                     cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem);
                     if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
+#endif
                     do {
                         U32 blockNb;
-                        size_t rSize;
                         for (blockNb=0; blockNb5) {
                                 int n;
                                 for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);

From ff8f83bd47d3e10152be2f622cfedf7c905c9881 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 20 Jun 2017 12:17:32 -0700
Subject: [PATCH 090/109] fixed fuzzer test

---
 tests/fuzzer.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index ee0aff08c..96336423c 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -420,6 +420,7 @@ static int basicUnitTests(U32 seed, double compressibility)
                 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
                 if (r != CNBuffSize - dictSize) goto _output_error;
             }
+            free(ddictBuffer);
             DISPLAYLEVEL(4, "OK (size of static DDict : %u) \n", (U32)ddictBufferSize);
         }
 

From b44ab82f7ad9c13d0f77d1fb279e4851a3eb47d4 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 20 Jun 2017 14:11:49 -0700
Subject: [PATCH 091/109] ensure new ZSTD_strategy starts at value 1

---
 lib/compress/zstd_compress.c   | 11 +++++++----
 lib/compress/zstdmt_compress.h | 13 +++++++------
 programs/zstd.1.md             |  4 ++--
 programs/zstdcli.c             |  2 +-
 4 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index f5ac75a95..1df1a4895 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2753,23 +2753,26 @@ static void ZSTD_compressBlock_btultra_extDict(ZSTD_CCtx* ctx, const void* src,
 }
 
 
+/* ZSTD_selectBlockCompressor() :
+ * assumption : strat is a valid strategy */
 typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize);
-
 static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
 {
     static const ZSTD_blockCompressor blockCompressor[2][(unsigned)ZSTD_btultra+1] = {
-        { NULL,
+        { ZSTD_compressBlock_fast  /* default for 0 */,
           ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy,
           ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2,
           ZSTD_compressBlock_btopt, ZSTD_compressBlock_btultra },
-        { NULL,
+        { ZSTD_compressBlock_fast_extDict  /* default for 0 */,
           ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict,
           ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict,
           ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict }
     };
     ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
+    assert((U32)strat >= (U32)ZSTD_fast);
+    assert((U32)strat <= (U32)ZSTD_btultra);
 
-    return blockCompressor[extDict][(U32)strat];
+    return blockCompressor[extDict!=0][(U32)strat];
 }
 
 
diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h
index 8b0ca824f..d36914de5 100644
--- a/lib/compress/zstdmt_compress.h
+++ b/lib/compress/zstdmt_compress.h
@@ -15,13 +15,15 @@
  #endif
 
 
-/* Note : All prototypes defined in this file must be considered experimental.
- *        There is no guarantee of API continuity on any of these prototypes */
+/* Note : All prototypes defined in this file are labelled experimental.
+ *        No guarantee of API continuity is provided on any of them.
+ *        In fact, the expectation is that these prototypes will be replaced
+ *        by ZSTD_compress_generic() API in the near future */
 
 /* ===   Dependencies   === */
-#include    /* size_t */
+#include                 /* size_t */
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_parameters */
-#include "zstd.h"     /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
+#include "zstd.h"            /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
 
 
 /* ===   Memory management   === */
@@ -32,7 +34,6 @@ ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads,
 ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx);
 
 ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx);
-ZSTDLIB_API size_t ZSTDMT_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned nbThreads);   /* not ready yet */
 
 
 /* ===   Simple buffer-to-butter one-pass function   === */
@@ -87,7 +88,7 @@ ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter
 
 /*! ZSTDMT_compressStream_generic() :
  *  Combines ZSTDMT_compressStream() with ZSTDMT_flushStream() or ZSTDMT_endStream()
- *  depending on flush directive
+ *  depending on flush directive.
  * @return : minimum amount of data still to be flushed
  *           0 if fully flushed
  *           or an error code */
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index 4e731ddba..63830586e 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -251,8 +251,8 @@ The list of available _options_:
     Specify a strategy used by a match finder.
 
     There are 8 strategies numbered from 0 to 7, from faster to stronger:
-    0=ZSTD\_fast, 1=ZSTD\_dfast, 2=ZSTD\_greedy, 3=ZSTD\_lazy,
-    4=ZSTD\_lazy2, 5=ZSTD\_btlazy2, 6=ZSTD\_btopt, 7=ZSTD\_btultra.
+    1=ZSTD\_fast, 2=ZSTD\_dfast, 3=ZSTD\_greedy, 4=ZSTD\_lazy,
+    5=ZSTD\_lazy2, 6=ZSTD\_btlazy2, 7=ZSTD\_btopt, 8=ZSTD\_btultra.
 
 - `windowLog`=_wlog_, `wlog`=_wlog_:
     Specify the maximum number of bits for a match distance.
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 32fef9993..f9f6fb56f 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -298,7 +298,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
         if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
-        if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         return 0;
     }

From 78b823455441adc67abe4f608e1dfa5dddfbb820 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 20 Jun 2017 14:26:48 -0700
Subject: [PATCH 092/109] fixed comments, following suggestion by @terrelln

---
 lib/zstd.h | 35 +++++++----------------------------
 1 file changed, 7 insertions(+), 28 deletions(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index e0c880103..bbfb92d55 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -422,7 +422,7 @@ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
 /* use this constant to defer to stdlib's functions */
-static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL};
+static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
 
 
 /***************************************
@@ -881,7 +881,6 @@ typedef enum {
                               * Special: value 0 means "do not change cLevel". */
     ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
                               * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
-                              * default value : set through compressionLevel.
                               * Special: value 0 means "do not change windowLog". */
     ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
                               * Resulting table size is (1 << (hashLog+2)).
@@ -974,12 +973,12 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
  *  Create an internal CDict from dict buffer.
  *  Decompression will have to use same buffer.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix,
+ *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
  *            meaning "return to no-dictionary mode".
  *  Note 1 : Dictionary content will be copied internally,
  *           except if ZSTD_p_refDictContent is set.
  *  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
- *           For this reason, compression parameters cannot be changed anymore after loading a prefix.
+ *           For this reason, compression parameters cannot be changed anymore after loading a dictionary.
  *           It's also a CPU-heavy operation, with non-negligible impact on latency.
  *  Note 3 : Dictionary will be used for all future compression jobs.
  *           To return to "no-dictionary" situation, load a NULL dictionary */
@@ -1018,17 +1017,17 @@ typedef enum {
     ZSTD_e_end         /* flush any remaining data and ends current frame. Any future compression starts a new frame. */
 } ZSTD_EndDirective;
 
-/*! ZSTD_compressStream_generic() :
+/*! ZSTD_compress_generic() :
  *  Behave about the same as ZSTD_compressStream. To note :
- *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_setCCtxParameter()
+ *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
  *  - Compression parameters cannot be changed once compression is started.
  *  - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
  *  - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
  *  - @return provides the minimum amount of data still to flush from internal buffers
  *            or an error code, which can be tested using ZSTD_isError().
- *            if @return != 0, flush is not fully completed, and must be called again to empty internal buffers.
+ *            if @return != 0, flush is not fully completed, there is some data left within internal buffers.
  *  - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
- *            only ZSTD_e_end and ZSTD_e_flush operations are allowed.
+ *            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
  *            It is necessary to fully flush internal buffers
  *            before starting a new compression job, or changing compression parameters.
  */
@@ -1061,26 +1060,6 @@ size_t ZSTD_compress_generic_simpleArgs (
                             ZSTD_EndDirective endOp);
 
 
-/* note : this part if completely experimental, not ready yet.
- * Main idea is : if ZSTD_parameters is not a good fit for stable API,
- * there will be a need for something else.
- * While initialising compression is now correctly covered,
- * creating an advanced CDict still needs to be solved.
- * Below is merely a plan of what it could look like */
-/*! ZSTD_CDict_createEmpty() :
- *  Create a CDict object which is still mutable after creation.
- *  It's the only one case allowing usage of ZSTD_CDict_setParameter().
- *  Once all compression parameters are selected,
- *  it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
- *  Dictionary content will be copied internally (except if ZSTD_p_refDictContent is set).
- *  After loading the dictionary, no more change is possible.
- *  The only remaining operation is to free CDict object.
- *  Note : An unfinished CDict behaves the same as a NULL CDict if referenced into a CCtx.
- */
-ZSTDLIB_API ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
-ZSTDLIB_API size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
-ZSTDLIB_API size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
-
 
 /**
     Block functions

From c3bce24ef4c86f780e94e047f1cbc4ec1963f4ed Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 20 Jun 2017 16:09:11 -0700
Subject: [PATCH 093/109] fixed potential dangling pointer, detected by
 @terrelln

---
 lib/common/zstd_internal.h       |  1 -
 lib/compress/zstd_compress.c     | 16 ++++++++--------
 lib/decompress/zstd_decompress.c | 13 +++++++------
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 77f6e5672..611c74f81 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -263,7 +263,6 @@ typedef struct {
 
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
 void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);
-int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
 
 /* custom memory allocation functions */
 void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 1df1a4895..7c4b526e2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -240,13 +240,13 @@ static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
     cctx->compressionLevel = ZSTD_CLEVEL_CUSTOM;
 }
 
+#define CLAMPCHECK(val,min,max) {                       \
+    if (((val)<(min)) | ((val)>(max))) {                \
+        return ERROR(compressionParameter_outOfBound);  \
+}   }
+
 size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
 {
-#   define CLAMPCHECK(val,min,max) {                         \
-        if ((valmax)) {                         \
-            return ERROR(compressionParameter_outOfBound);  \
-    }   }
-
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
 
     switch(param)
@@ -3155,7 +3155,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict,
 }
 
 /*! ZSTD_compressBegin_internal() :
-*   @return : 0, or an error code */
+ * @return : 0, or an error code */
 size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
                              const ZSTD_CDict* cdict,
@@ -3347,10 +3347,10 @@ static size_t ZSTD_initCDict_internal(
         cdict->dictContent = dictBuffer;
     } else {
         void* const internalBuffer = ZSTD_malloc(dictSize, cdict->refContext->customMem);
-        if (!internalBuffer) return ERROR(memory_allocation);
-        memcpy(internalBuffer, dictBuffer, dictSize);
         cdict->dictBuffer = internalBuffer;
         cdict->dictContent = internalBuffer;
+        if (!internalBuffer) return ERROR(memory_allocation);
+        memcpy(internalBuffer, dictBuffer, dictSize);
     }
     cdict->dictContentSize = dictSize;
 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index aeaa40c1f..98372f94c 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1688,21 +1688,22 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
     }
 }
 
-int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }   /* for zbuff */
+static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }
 
 /** ZSTD_decompressContinue() :
-*   @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
-*             or an error code, which can be tested using ZSTD_isError() */
+ *  srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress())
+ *  @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
+ *            or an error code, which can be tested using ZSTD_isError() */
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     /* Sanity check */
-    if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
+    if (srcSize != dctx->expected) return ERROR(srcSize_wrong);   /* unauthorized */
     if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
 
     switch (dctx->stage)
     {
     case ZSTDds_getFrameHeaderSize :
-        if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);      /* impossible */
+        if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);      /* unauthorized */
         if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {        /* skippable frame */
             memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
             dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix;  /* magic number + skippable frame length */
@@ -1747,7 +1748,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
                     dctx->stage = ZSTDds_getFrameHeaderSize;
                 }
             } else {
-                dctx->expected = 3;  /* go directly to next header */
+                dctx->expected = ZSTD_blockHeaderSize;  /* jump to next header */
                 dctx->stage = ZSTDds_decodeBlockHeader;
             }
             return 0;

From 466f92eaa643271ec1b537bc11e47bda5c96c121 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 20 Jun 2017 16:25:29 -0700
Subject: [PATCH 094/109] removed one useless streaming compression stage,
 detected by @terrelln

---
 lib/compress/zstd_compress.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 7c4b526e2..e48bc41bd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -68,7 +68,7 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
 /*-*************************************
 *  Context memory management
 ***************************************/
-typedef enum { zcss_init=0, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
+typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
 
 struct ZSTD_CDict_s {
     void* dictBuffer;
@@ -3371,7 +3371,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
     DEBUGLOG(5, "ZSTD_createCDict_advanced");
     if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
 
-    {   ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
+    {   ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
         ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
 
         if (!cdict || !cctx) {
@@ -3588,15 +3588,16 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
             return ERROR(memory_allocation);
         }
         ZSTD_freeCDict(zcs->cdictLocal);
-        zcs->cdict = NULL;
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params.cParams, zcs->customMem);
-        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
         zcs->cdict = zcs->cdictLocal;
+        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
     } else {
         if (cdict) {
             ZSTD_parameters const cdictParams = ZSTD_getParamsFromCDict(cdict);
             params.cParams = cdictParams.cParams;  /* cParams are enforced from cdict */
         }
+        ZSTD_freeCDict(zcs->cdictLocal);
+        zcs->cdictLocal = NULL;
         zcs->cdict = cdict;
     }
 
@@ -3779,9 +3780,6 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 break;
             }
 
-        case zcss_final:
-            someMoreWork = 0; break;  /* useless */
-
         default: /* impossible */
             assert(0);
         }

From e51d51bdf7d77b66ea4beb87a52097633147b2a9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 20 Jun 2017 17:44:55 -0700
Subject: [PATCH 095/109] fixed memcpy() overlap

---
 lib/compress/zstd_compress.c     |  5 ++---
 lib/decompress/zstd_decompress.c | 11 +++++------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e48bc41bd..4f1a0bcc4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -4017,14 +4017,13 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
 *   Size values are optional, provide 0 if not known or unused */
 ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
 {
-    ZSTD_compressionParameters cp;
     size_t const addedSize = srcSizeHint ? 0 : 500;
     U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1;
     U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);   /* intentional underflow for srcSizeHint == 0 */
     if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* 0 == default; no negative compressionLevel yet */
     if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
-    cp = ZSTD_defaultCParameters[tableID][compressionLevel];
-    return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize);   /* no need to ensure initial CParams validity */
+    { ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];
+      return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); }
 }
 
 /*! ZSTD_getParams() :
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 98372f94c..51fa244ce 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2306,14 +2306,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     break;
             }   }
 
-            /* Consume header */
+            /* Consume header (see ZSTDds_decodeFrameHeader) */
             CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
-            {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds);  /* == ZSTD_frameHeaderSize_prefix */
-                CHECK_F(ZSTD_decompressContinue(zds, NULL, 0, zds->headerBuffer, h1Size));
-                {   size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds);
-                    CHECK_F(ZSTD_decompressContinue(zds, NULL, 0, zds->headerBuffer+h1Size, h2Size));
-            }   }
+            CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
+            zds->expected = ZSTD_blockHeaderSize;
+            zds->stage = ZSTDds_decodeBlockHeader;
 
+            /* control buffer memory usage */
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
             if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
 

From 7bd1a2900e8e2713cdc9801a9ff81563f39aec9c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 11:50:33 -0700
Subject: [PATCH 096/109] added ZSTD_dictMode_e to control dictionary loading
 mode

---
 doc/zstd_manual.html              |  47 +++++------
 lib/common/zstd_internal.h        |   3 +-
 lib/compress/zstd_compress.c      | 125 ++++++++++++++++++++----------
 lib/compress/zstdmt_compress.c    |   5 +-
 lib/zstd.h                        |  29 ++++---
 programs/bench.c                  |   2 +-
 tests/fuzzer.c                    |  71 +++++++++++------
 tests/zstreamtest.c               |   2 +-
 zlibWrapper/examples/zwrapbench.c |   4 +-
 9 files changed, 177 insertions(+), 111 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 88e890793..63704e67e 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -341,7 +341,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
     /* use this constant to defer to stdlib's functions */
-static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL};
+static const ZSTD_customMem ZSTD_defaultCMem = { NULL, NULL, NULL };
 

    
 
    Frame size functions
    
 
@@ -461,14 +461,18 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
   It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict 
 

    
 
-
    ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
+
    typedef enum { ZSTD_dm_auto=0, ZSTD_dm_rawContent, ZSTD_dm_fullDict } ZSTD_dictMode_e;
+

    
+
    ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+                                      unsigned byReference, ZSTD_dictMode_e dictMode,
                                       ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
 
      Create a ZSTD_CDict using external alloc and free, and customized compression parameters 
 

    
 
 
    ZSTD_CDict* ZSTD_initStaticCDict(
                 void* workspace, size_t workspaceSize,
-          const void* dict, size_t dictSize, unsigned byReference,
+          const void* dict, size_t dictSize,
+                unsigned byReference, ZSTD_dictMode_e dictMode,
                 ZSTD_compressionParameters cParams);
 
      Generate a digested dictionary in provided memory area.
   workspace: The memory area to emplace the dictionary into.
@@ -743,7 +747,6 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
                               * Special: value 0 means "do not change cLevel". */
     ZSTD_p_windowLog,        /* Maximum allowed back-reference distance, expressed as power of 2.
                               * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
-                              * default value : set through compressionLevel.
                               * Special: value 0 means "do not change windowLog". */
     ZSTD_p_hashLog,          /* Size of the probe table, as a power of 2.
                               * Resulting table size is (1 << (hashLog+2)).
@@ -791,10 +794,10 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
     /* dictionary parameters */
     ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
                               * This avoids duplicating dictionary content.
-                              * But it also requires that dictionary buffer outlives its user (CDict) */
-                             /* Not ready yet ! */
-    ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
-                             /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
+                              * But it also requires that dictionary buffer outlives its users */
+                             /* Not ready yet ! <=================================== */
+    ZSTD_p_dictMode,         /* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.
+                              * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
 
     /* multi-threading parameters */
     ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
@@ -810,7 +813,7 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
 
     /* advanced parameters - may not remain available after API update */
     ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
-                              * even when referencing into Dictionary content
+                              * even when referencing into Dictionary content.
                               * default : 0 when using a CDict, 1 when using a Prefix */
 } ZSTD_cParameter;
 

    
@@ -835,12 +838,12 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
 
      Create an internal CDict from dict buffer.
   Decompression will have to use same buffer.
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
-  Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix,
+  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
             meaning "return to no-dictionary mode".
   Note 1 : Dictionary content will be copied internally,
            except if ZSTD_p_refDictContent is set.
   Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
-           For this reason, compression parameters cannot be changed anymore after loading a prefix.
+           For this reason, compression parameters cannot be changed anymore after loading a dictionary.
            It's also a CPU-heavy operation, with non-negligible impact on latency.
   Note 3 : Dictionary will be used for all future compression jobs.
            To return to "no-dictionary" situation, load a NULL dictionary 
@@ -859,7 +862,7 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
  
 

    
 
-
    size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
+
    size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! <===================================== */
 
      Reference a prefix (content-only dictionary) to bootstrap next compression job.
   Decompression will have to use same prefix.
   Prefix is only used once. Tables are discarded at end of compression job.
@@ -882,15 +885,15 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
                               ZSTD_inBuffer* input,
                               ZSTD_EndDirective endOp);
 
      Behave about the same as ZSTD_compressStream. To note :
-  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_setCCtxParameter()
+  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
   - Compression parameters cannot be changed once compression is started.
   - *dstPos must be <= dstCapacity, *srcPos must be <= srcSize
   - *dspPos and *srcPos will be updated. They are guaranteed to remain below their respective limit.
   - @return provides the minimum amount of data still to flush from internal buffers
             or an error code, which can be tested using ZSTD_isError().
-            if @return != 0, flush is not fully completed, and must be called again to empty internal buffers.
+            if @return != 0, flush is not fully completed, there is some data left within internal buffers.
   - after a ZSTD_e_end directive, if internal buffer is not fully flushed,
-            only ZSTD_e_end and ZSTD_e_flush operations are allowed.
+            only ZSTD_e_end or ZSTD_e_flush operations are allowed.
             It is necessary to fully flush internal buffers
             before starting a new compression job, or changing compression parameters.
  
@@ -918,20 +921,6 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
  
 

    
 
-
    ZSTD_CDict* ZSTD_CDict_createEmpty(void);   /* Not ready yet ! */
-size_t ZSTD_CDict_setParameter(ZSTD_CDict* cdict, ZSTD_cParameter param, unsigned value);  /* Not ready yet ! */
-size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dictSize);    /* Not ready yet ! */
-
      Create a CDict object which is still mutable after creation.
-  It's the only one case allowing usage of ZSTD_CDict_setParameter().
-  Once all compression parameters are selected,
-  it's possible to load the target dictionary, using ZSTD_CDict_loadDictionary().
-  Dictionary content will be copied internally (except if ZSTD_p_refDictContent is set).
-  After loading the dictionary, no more change is possible.
-  The only remaining operation is to free CDict object.
-  Note : An unfinished CDict behaves the same as a NULL CDict if referenced into a CCtx.
- 
-

    
-
 
    Block functions
         Block functions produce and decode raw zstd blocks, without frame metadata.
     Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 611c74f81..4e216d728 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -305,6 +305,7 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
 
 
 typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e;
+#if 0
 /*! ZSTD_compressBegin_internal() :
  *  innermost initialization function. Private use only.
  *  expects params to be valid.
@@ -315,7 +316,7 @@ size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                              const ZSTD_CDict* cdict,
                                    ZSTD_parameters params, U64 pledgedSrcSize,
                                    ZSTD_buffered_policy_e zbuff);
-
+#endif
 
 /*! ZSTD_initCStream_internal() :
  *  Private use only. Init streaming operation.
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 4f1a0bcc4..59ac3f35d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -88,7 +88,7 @@ struct ZSTD_CCtx_s {
     U32   hashLog3;         /* dispatch table : larger == faster, more memory */
     U32   loadedDictEnd;    /* index of end of dictionary */
     U32   forceWindow;      /* force back-references to respect limit of 1<staticSize = workspaceSize;
     cctx->workSpace = (void*)(cctx+1);
     cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx);
 
     /* entropy space (never moves) */
-    /* note : this code should be shared with resetCCtx, instead of copied */
+    /* note : this code should be shared with resetCCtx, rather than copy/pasted */
     {   void* ptr = cctx->workSpace;
         cctx->hufCTable = (HUF_CElt*)ptr;
         ptr = (char*)cctx->hufCTable + hufCTable_size;
@@ -203,6 +203,10 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
 size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
 {
     if (cctx==NULL) return 0;   /* support sizeof on NULL */
+    DEBUGLOG(5, "sizeof(*cctx) : %u", (U32)sizeof(*cctx));
+    DEBUGLOG(5, "workSpaceSize : %u", (U32)cctx->workSpaceSize);
+    DEBUGLOG(5, "streaming buffers : %u", (U32)(cctx->outBuffSize + cctx->inBuffSize));
+    DEBUGLOG(5, "inner MTCTX : %u", (U32)ZSTDMT_sizeof_CCtx(cctx->mtctx));
     return sizeof(*cctx) + cctx->workSpaceSize
            + ZSTD_sizeof_CDict(cctx->cdictLocal)
            + cctx->outBuffSize + cctx->inBuffSize
@@ -225,7 +229,9 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
     switch(param)
     {
     case ZSTD_p_forceWindow : cctx->forceWindow = value>0; cctx->loadedDictEnd = 0; return 0;
-    case ZSTD_p_forceRawDict : cctx->forceRawDict = value>0; return 0;
+    ZSTD_STATIC_ASSERT(ZSTD_dm_auto==0);
+    ZSTD_STATIC_ASSERT(ZSTD_dm_rawContent==1);
+    case ZSTD_p_forceRawDict : cctx->dictMode = (ZSTD_dictMode_e)(value>0); return 0;
     default: return ERROR(parameter_unknown);
     }
 }
@@ -340,6 +346,14 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
     case ZSTD_p_refDictContent :   /* to be done later */
         return ERROR(compressionParameter_unsupported);
 
+    case ZSTD_p_dictMode :
+        /* restrict dictionary mode, to "rawContent" or "fullDict" only */
+        ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
+        if (value > (unsigned)ZSTD_dm_fullDict)
+            return ERROR(compressionParameter_outOfBound);
+        cctx->dictMode = (ZSTD_dictMode_e)value;
+        return 0;
+
     case ZSTD_p_forceMaxWindow :  /* Force back-references to remain < windowSize,
                                    * even when referencing into Dictionary content
                                    * default : 0 when using a CDict, 1 when using a Prefix */
@@ -375,10 +389,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         assert(cctx->mtctx != NULL);
         return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
 
-    case ZSTD_p_rawContentDict :  /* load dictionary in "content-only" mode (no header analysis) (default:0) */
-        cctx->forceRawDict = value>0;
-        return 0;
-
     default: return ERROR(parameter_unknown);
     }
 }
@@ -407,7 +417,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
                 ZSTD_getCParams(cctx->compressionLevel, 0, dictSize);
         cctx->cdictLocal = ZSTD_createCDict_advanced(
                                 dict, dictSize,
-                                0 /* byReference */,
+                                0 /* byReference */, cctx->dictMode,
                                 cParams, cctx->customMem);
         cctx->cdict = cctx->cdictLocal;
         if (cctx->cdictLocal == NULL)
@@ -543,6 +553,8 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
     size_t const optSpace = ((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btultra)) ? optBudget : 0;
     size_t const neededSpace = entropySpace + tableSpace + tokenSpace + optSpace;
 
+    DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx));
+    DEBUGLOG(5, "estimate workSpace : %u", (U32)neededSpace);
     return sizeof(ZSTD_CCtx) + neededSpace;
 }
 
@@ -625,8 +637,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                                 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
         size_t const h3Size = ((size_t)1) << hashLog3;
         size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
-        size_t const buffOutSize = ZSTD_compressBound(blockSize)+1;
-        size_t const buffInSize = ((size_t)1 << params.cParams.windowLog) + blockSize;
+        size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0;
+        size_t const buffInSize = (zbuff==ZSTDb_buffered) ? ((size_t)1 << params.cParams.windowLog) + blockSize : 0;
         void* ptr;
 
         /* Check if workSpace is large enough, alloc a new one if needed */
@@ -638,8 +650,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
             size_t const optSpace = ( (params.cParams.strategy == ZSTD_btopt)
                                     || (params.cParams.strategy == ZSTD_btultra)) ?
                                     optPotentialSpace : 0;
-            size_t const bufferSpace = (zbuff==ZSTDb_buffered) ?
-                                        buffInSize + buffOutSize : 0;
+            size_t const bufferSpace = buffInSize + buffOutSize;
             size_t const neededSpace = entropySpace + optSpace + tableSpace
                                      + tokenSpace + bufferSpace;
 
@@ -3142,22 +3153,33 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t
 
 /** ZSTD_compress_insertDictionary() :
 *   @return : 0, or an error code */
-static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
+static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
+                                       const void* dict, size_t dictSize,
+                                             ZSTD_dictMode_e dictMode)
 {
     if ((dict==NULL) || (dictSize<=8)) return 0;
 
-    /* dict as pure content */
-    if ((MEM_readLE32(dict) != ZSTD_DICT_MAGIC) || (cctx->forceRawDict))
+    /* dict restricted modes */
+    if (dictMode==ZSTD_dm_rawContent)
         return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
 
-    /* dict as zstd dictionary */
+    if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) {
+        if (dictMode == ZSTD_dm_auto)
+            return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
+        if (dictMode == ZSTD_dm_fullDict)
+            return ERROR(dictionary_wrong);
+        assert(0);   /* impossible */
+    }
+
+    /* dict as full zstd dictionary */
     return ZSTD_loadZstdDictionary(cctx, dict, dictSize);
 }
 
 /*! ZSTD_compressBegin_internal() :
  * @return : 0, or an error code */
-size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
+static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
+                             ZSTD_dictMode_e dictMode,
                              const ZSTD_CDict* cdict,
                                    ZSTD_parameters params, U64 pledgedSrcSize,
                                    ZSTD_buffered_policy_e zbuff)
@@ -3172,7 +3194,7 @@ size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
 
     CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                     ZSTDcrp_continue, zbuff));
-    return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
+    return ZSTD_compress_insertDictionary(cctx, dict, dictSize, dictMode);
 }
 
 
@@ -3184,7 +3206,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
 {
     /* compression parameters verification and optimization */
     CHECK_F(ZSTD_checkCParams(params.cParams));
-    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
+    return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
                                     params, pledgedSrcSize, ZSTDb_not_buffered);
 }
 
@@ -3192,7 +3214,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
-    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
+    return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
                                        params, 0, ZSTDb_not_buffered);
 }
 
@@ -3273,8 +3295,8 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
                          const void* dict,size_t dictSize,
                                ZSTD_parameters params)
 {
-    CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
-                                        params, srcSize, ZSTDb_not_buffered));
+    CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
+                                         params, srcSize, ZSTDb_not_buffered) );
     return ZSTD_compressEnd(cctx, dst,  dstCapacity, src, srcSize);
 }
 
@@ -3319,6 +3341,8 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
  *  Estimate amount of memory that will be needed to create a dictionary with following arguments */
 size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
 {
+    DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict));
+    DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize(cParams));
     return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize(cParams)
            + (byReference ? 0 : dictSize);
 }
@@ -3326,6 +3350,8 @@ size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSiz
 size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
 {
     if (cdict==NULL) return 0;   /* support sizeof on NULL */
+    DEBUGLOG(5, "sizeof(*cdict) : %u", (U32)sizeof(*cdict));
+    DEBUGLOG(5, "ZSTD_sizeof_CCtx : %u", (U32)ZSTD_sizeof_CCtx(cdict->refContext));
     return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict);
 }
 
@@ -3339,7 +3365,8 @@ static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_
 
 static size_t ZSTD_initCDict_internal(
                     ZSTD_CDict* cdict,
-              const void* dictBuffer, size_t dictSize, unsigned byReference,
+              const void* dictBuffer, size_t dictSize,
+                    unsigned byReference, ZSTD_dictMode_e dictMode,
                     ZSTD_compressionParameters cParams)
 {
     if ((byReference) || (!dictBuffer) || (!dictSize)) {
@@ -3357,15 +3384,18 @@ static size_t ZSTD_initCDict_internal(
     {   ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */,
                     0 /* checksumFlag */, 0 /* noDictIDFlag */ };  /* dummy */
         ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams);
-        CHECK_F( ZSTD_compressBegin_advanced(cdict->refContext,
-                                            cdict->dictContent, dictSize,
-                                            params, 0 /* srcSize */) );
+        CHECK_F( ZSTD_compressBegin_internal(cdict->refContext,
+                                        cdict->dictContent, dictSize, dictMode,
+                                        NULL,
+                                        params, ZSTD_CONTENTSIZE_UNKNOWN,
+                                        ZSTDb_not_buffered) );
     }
 
     return 0;
 }
 
-ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
+ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
+                                      unsigned byReference, ZSTD_dictMode_e dictMode,
                                       ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
 {
     DEBUGLOG(5, "ZSTD_createCDict_advanced");
@@ -3382,7 +3412,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
         cdict->refContext = cctx;
 
         if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
-                                        dictBuffer, dictSize, byReference,
+                                        dictBuffer, dictSize,
+                                        byReference, dictMode,
                                         cParams) )) {
             ZSTD_freeCDict(cdict);
             return NULL;
@@ -3394,16 +3425,18 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
 
 ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_customMem const allocator = { NULL, NULL, NULL };
     ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
-    return ZSTD_createCDict_advanced(dict, dictSize, 0, cParams, allocator);
+    return ZSTD_createCDict_advanced(dict, dictSize,
+                                     0 /* byReference */, ZSTD_dm_auto,
+                                     cParams, ZSTD_defaultCMem);
 }
 
 ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_customMem const allocator = { NULL, NULL, NULL };
     ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
-    return ZSTD_createCDict_advanced(dict, dictSize, 1, cParams, allocator);
+    return ZSTD_createCDict_advanced(dict, dictSize,
+                                     1 /* byReference */, ZSTD_dm_auto,
+                                     cParams, ZSTD_defaultCMem);
 }
 
 size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
@@ -3431,7 +3464,8 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
  *         Since workspace was allocated externally, it must be freed externally.
  */
 ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
-                           const void* dict, size_t dictSize, unsigned byReference,
+                           const void* dict, size_t dictSize,
+                                 unsigned byReference, ZSTD_dictMode_e dictMode,
                                  ZSTD_compressionParameters cParams)
 {
     size_t const cctxSize = ZSTD_estimateCCtxSize(cParams);
@@ -3455,8 +3489,9 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
     cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize);
 
     if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
-                                        dict, dictSize, 1 /* by Reference */,
-                                        cParams) ))
+                                              dict, dictSize,
+                                              1 /* byReference */, dictMode,
+                                              cParams) ))
         return NULL;
 
     return cdict;
@@ -3476,8 +3511,11 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     {   ZSTD_parameters params = cdict->refContext->appliedParams;
         params.fParams = fParams;
         DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
-        return ZSTD_compressBegin_internal(cctx, NULL, 0, cdict,
-                                params, pledgedSrcSize, ZSTDb_not_buffered);
+        return ZSTD_compressBegin_internal(cctx,
+                                           NULL, 0, ZSTD_dm_auto,
+                                           cdict,
+                                           params, pledgedSrcSize,
+                                           ZSTDb_not_buffered);
     }
 }
 
@@ -3552,8 +3590,11 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
 {
     DEBUGLOG(5, "ZSTD_resetCStream_internal");
 
-    CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, zcs->cdict,
-                                    params, pledgedSrcSize, ZSTDb_buffered));
+    CHECK_F( ZSTD_compressBegin_internal(zcs,
+                                        NULL, 0, ZSTD_dm_auto,
+                                        zcs->cdict,
+                                        params, pledgedSrcSize,
+                                        ZSTDb_buffered) );
 
     zcs->inToCompress = 0;
     zcs->inBuffPos = 0;
@@ -3588,7 +3629,9 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
             return ERROR(memory_allocation);
         }
         ZSTD_freeCDict(zcs->cdictLocal);
-        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params.cParams, zcs->customMem);
+        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
+                                                    0 /* byReference */, ZSTD_dm_auto,
+                                                    params.cParams, zcs->customMem);
         zcs->cdict = zcs->cdictLocal;
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
     } else {
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 895d362fb..499c531c7 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -584,8 +584,9 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     if (dict) {
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdict = NULL;
-        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* byRef */,
-                                                params.cParams, zcs->cMem);
+        zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
+                                                    0 /* byRef */, ZSTD_dm_auto,
+                                                    params.cParams, zcs->cMem);
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
         zcs->cdict = zcs->cdictLocal;
     } else {
diff --git a/lib/zstd.h b/lib/zstd.h
index bbfb92d55..30972cd0b 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -552,9 +552,12 @@ ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter par
  *  It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
+
+typedef enum { ZSTD_dm_auto=0, ZSTD_dm_rawContent, ZSTD_dm_fullDict } ZSTD_dictMode_e;
 /*! ZSTD_createCDict_advanced() :
  *  Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
-ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
+ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+                                                  unsigned byReference, ZSTD_dictMode_e dictMode,
                                                   ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
 
 /*! ZSTD_initStaticCDict_advanced() :
@@ -572,7 +575,8 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
  */
 ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict(
                             void* workspace, size_t workspaceSize,
-                      const void* dict, size_t dictSize, unsigned byReference,
+                      const void* dict, size_t dictSize,
+                            unsigned byReference, ZSTD_dictMode_e dictMode,
                             ZSTD_compressionParameters cParams);
 
 /*! ZSTD_getCParams() :
@@ -867,13 +871,16 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
 /* note on naming convention :
  *   Initially, the API favored names like ZSTD_setCCtxParameter() .
  *   In this proposal, convention is changed towards ZSTD_CCtx_setParameter() .
- *   The main idea is that it identifies more clearly the target object type.
- *   It feels clearer when considering other potential variants :
+ *   The main driver is that it identifies more clearly the target object type.
+ *   It feels clearer in light of potential variants :
  *   ZSTD_CDict_setParameter() (rather than ZSTD_setCDictParameter())
  *   ZSTD_DCtx_setParameter()  (rather than ZSTD_setDCtxParameter() )
- *   The left variant feels clearer.
+ *   Left variant feels easier to distinguish.
  */
 
+/* note on enum design :
+ * All enum will be manually set to explicit values before reaching "stable API" status */
+
 typedef enum {
     /* compression parameters */
     ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table
@@ -928,10 +935,10 @@ typedef enum {
     /* dictionary parameters */
     ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
                               * This avoids duplicating dictionary content.
-                              * But it also requires that dictionary buffer outlives its user (CDict) */
-                             /* Not ready yet ! */
-    ZSTD_p_rawContentDict,   /* load dictionary in "content-only" mode (no header analysis) (default:0) */
-                             /* question : should there be an option to load dictionary only in zstd format, rejecting others with an error code ? */
+                              * But it also requires that dictionary buffer outlives its users */
+                             /* Not ready yet ! <=================================== */
+    ZSTD_p_dictMode,         /* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.
+                              * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
 
     /* multi-threading parameters */
     ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
@@ -947,7 +954,7 @@ typedef enum {
 
     /* advanced parameters - may not remain available after API update */
     ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
-                              * even when referencing into Dictionary content
+                              * even when referencing into Dictionary content.
                               * default : 0 when using a CDict, 1 when using a Prefix */
 } ZSTD_cParameter;
 
@@ -1007,7 +1014,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
  *  Note 1 : Prefix buffer is referenced. It must outlive compression job.
  *  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
  *           It's a CPU-heavy operation, with non-negligible impact on latency. */
-ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! */
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);   /* Not ready yet ! <===================================== */
 
 
 
diff --git a/programs/bench.c b/programs/bench.c
index 6f09a56de..f9493e3b0 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -284,7 +284,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                     if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
                     if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
                     if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
-                    cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem);
+                    cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1 /*byRef*/, ZSTD_dm_auto, zparams.cParams, cmem);
                     if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
 #endif
                     do {
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 96336423c..a3a56d9d8 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -502,15 +502,16 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "test%3i : estimate CDict size : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
-            size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize, 1);
+            size_t const estimatedSize = ZSTD_estimateCDictSize(cParams, dictSize, 1 /*byReference*/);
             DISPLAYLEVEL(4, "OK : %u \n", (U32)estimatedSize);
         }
 
-        DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
+        DISPLAYLEVEL(4, "test%3i : compress with CDict ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
-            ZSTD_customMem const customMem = { NULL, NULL, NULL };
             ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
-                                    1 /* by Referece */, cParams, customMem);
+                                            1 /* byReference */, ZSTD_dm_auto,
+                                            cParams, ZSTD_defaultCMem);
+            DISPLAYLEVEL(4, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict));
             cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
                                                  CNBuffer, CNBuffSize, cdict);
             ZSTD_freeCDict(cdict);
@@ -538,7 +539,8 @@ static int basicUnitTests(U32 seed, double compressibility)
             void* const cdictBuffer = malloc(cdictSize);
             if (cdictBuffer==NULL) goto _output_error;
             {   ZSTD_CDict* const cdict = ZSTD_initStaticCDict(cdictBuffer, cdictSize,
-                                            dictBuffer, dictSize, 0 /* by Reference */,
+                                            dictBuffer, dictSize,
+                                            0 /* by Reference */, ZSTD_dm_auto,
                                             cParams);
                 if (cdict == NULL) {
                     DISPLAY("ZSTD_initStaticCDict failed ");
@@ -558,8 +560,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
         {   ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
             ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
-            ZSTD_customMem const customMem = { NULL, NULL, NULL };
-            ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
+            ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1 /*byRef*/, ZSTD_dm_auto, cParams, ZSTD_defaultCMem);
             cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
                                                  CNBuffer, CNBuffSize, cdict, fParams);
             ZSTD_freeCDict(cdict);
@@ -608,6 +609,22 @@ static int basicUnitTests(U32 seed, double compressibility)
         }
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
+        {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
+            ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1 /*byRef*/, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
+            if (cdict==NULL) goto _output_error;
+            ZSTD_freeCDict(cdict);
+        }
+        DISPLAYLEVEL(4, "OK \n");
+
+        DISPLAYLEVEL(4, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
+        {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
+            ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, 1 /*byRef*/, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem);
+            if (cdict!=NULL) goto _output_error;
+            ZSTD_freeCDict(cdict);
+        }
+        DISPLAYLEVEL(4, "OK \n");
+
         ZSTD_freeCCtx(cctx);
         free(dictBuffer);
         free(samplesSizes);
@@ -686,9 +703,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         size_t const wrongSrcSize = (srcSize + 1000);
         ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
         params.fParams.contentSizeFlag = 1;
-        {   size_t const result = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize);
-            if (ZSTD_isError(result)) goto _output_error;
-        }
+        CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
         {   size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
             if (!ZSTD_isError(result)) goto _output_error;
             if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
@@ -870,8 +885,23 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
 }
 
 #undef CHECK
-#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
-                         DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb); goto _output_error; }
+#define CHECK(cond, ...) {                                    \
+    if (cond) {                                               \
+        DISPLAY("Error => ");                                 \
+        DISPLAY(__VA_ARGS__);                                 \
+        DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb);  \
+        goto _output_error;                                   \
+}   }
+
+#define CHECK_Z(f) {                                          \
+    size_t const err = f;                                     \
+    if (ZSTD_isError(err)) {                                  \
+        DISPLAY("Error => %s : %s ",                          \
+                #f, ZSTD_getErrorName(err));                  \
+        DISPLAY(" (seed %u, test nb %u)  \n", seed, testNb);  \
+        goto _output_error;                                   \
+}   }
+
 
 static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
 {
@@ -984,8 +1014,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
 
         /* frame header decompression test */
         {   ZSTD_frameHeader dParams;
-            size_t const check = ZSTD_getFrameHeader(&dParams, cBuffer, cSize);
-            CHECK(ZSTD_isError(check), "Frame Parameters extraction failed");
+            CHECK_Z( ZSTD_getFrameHeader(&dParams, cBuffer, cSize) );
             CHECK(dParams.frameContentSize != sampleSize, "Frame content size incorrect");
         }
 
@@ -1072,20 +1101,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
             dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
 
             if (FUZ_rand(&lseed) & 0xF) {
-                size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
-                CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
+                CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
             } else {
                 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
                 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
                                                     !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
                                                     0 /*NodictID*/ };   /* note : since dictionary is fake, dictIDflag has no impact */
                 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
-                size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
-                CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
+                CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
             }
-            {   size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0);
-                CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
-        }   }
+            CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
+        }
         ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1);
 
         {   U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
@@ -1117,8 +1143,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
 
         /* streaming decompression test */
         if (dictSize<8) dictSize=0, dict=NULL;   /* disable dictionary */
-        { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
-          CHECK (ZSTD_isError(errorCode), "ZSTD_decompressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
+        CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
         totalCSize = 0;
         totalGenSize = 0;
         while (totalCSize < cSize) {
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 95b3add0b..7229f7031 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -486,7 +486,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
         ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
-        ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1 /* byReference */, cParams, customMem);
+        ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, 1 /* byReference */, ZSTD_dm_auto, cParams, customMem);
         size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
         if (ZSTD_isError(initError)) goto _output_error;
         cSize = 0;
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
index a57ed51ec..1fc2117f6 100644
--- a/zlibWrapper/examples/zwrapbench.c
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -90,7 +90,7 @@ static clock_t g_time = 0;
 #ifndef DEBUG
 #  define DEBUG 0
 #endif
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
 #define EXM_THROW(error, ...)                                             \
 {                                                                         \
     DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
@@ -236,7 +236,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                 if (compressor == BMK_ZSTD) {
                     ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
                     ZSTD_customMem const cmem = { NULL, NULL, NULL };
-                    ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams.cParams, cmem);
+                    ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1 /*byRef*/, ZSTD_dm_auto, zparams.cParams, cmem);
                     if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
 
                     do {

From 83095970e699c2430dc2266f2472e1393aaf0822 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 12:26:40 -0700
Subject: [PATCH 097/109] free cdictLocal faster, suggested by @terrelln

---
 lib/compress/zstdmt_compress.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 499c531c7..f07143c1a 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -583,13 +583,14 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
     zcs->frameContentSize = pledgedSrcSize;
     if (dict) {
         ZSTD_freeCDict(zcs->cdictLocal);
-        zcs->cdict = NULL;
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
                                                     0 /* byRef */, ZSTD_dm_auto,
                                                     params.cParams, zcs->cMem);
-        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
         zcs->cdict = zcs->cdictLocal;
+        if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
     } else {
+        ZSTD_freeCDict(zcs->cdictLocal);
+        zcs->cdictLocal = NULL;
         zcs->cdict = cdict;
     }
 
@@ -627,7 +628,7 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
                                      unsigned long long pledgedSrcSize)
 {
     ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
-    if (cdict==NULL) return ERROR(GENERIC);   /* method incompatible with NULL cdict */
+    if (cdict==NULL) return ERROR(dictionary_wrong);   /* method incompatible with NULL cdict */
     params.fParams = fParams;
     return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, cdict,
                                         params, pledgedSrcSize);

From 1e4129b27b2dfa840b6396c5f8b6222ac404bcd8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 13:26:10 -0700
Subject: [PATCH 098/109] fixed dangling pointer risk, detected by @terrelln

---
 lib/decompress/zstd_decompress.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 51fa244ce..4dd78e0e5 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -223,6 +223,7 @@ ZSTD_DCtx* ZSTD_createDCtx(void)
 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
 {
     if (dctx==NULL) return 0;   /* support free on NULL */
+    if (dctx->staticSize) return ERROR(memory_allocation);   /* not compatible with static DCtx */
     {   ZSTD_customMem const cMem = dctx->customMem;
         ZSTD_freeDDict(dctx->ddictLocal);
         dctx->ddictLocal = NULL;
@@ -237,7 +238,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
     }
 }
 
-/* no longer appropriate */
+/* no longer useful */
 void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 {
     size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
@@ -1981,10 +1982,10 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_
         ddict->dictContent = dict;
     } else {
         void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
-        if (!internalBuffer) return ERROR(memory_allocation);
-        memcpy(internalBuffer, dict, dictSize);
         ddict->dictBuffer = internalBuffer;
         ddict->dictContent = internalBuffer;
+        if (!internalBuffer) return ERROR(memory_allocation);
+        memcpy(internalBuffer, dict, dictSize);
     }
     ddict->dictSize = dictSize;
     ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
@@ -2353,7 +2354,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     break;
                 }
                 if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
-                    const int isSkipFrame = ZSTD_isSkipFrame(zds);
+                    int const isSkipFrame = ZSTD_isSkipFrame(zds);
                     size_t const decodedSize = ZSTD_decompressContinue(zds,
                         zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
                         ip, neededInSize);

From 204b6b7ef6f5afe0cd67700012debc9ce981b0c2 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 15:13:00 -0700
Subject: [PATCH 099/109] fixed streaming buffered allocation with CDict
 compression

---
 lib/compress/zstd_compress.c   | 41 +++++++++++++++++++++++-----------
 lib/compress/zstdmt_compress.c |  3 +++
 2 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 59ac3f35d..06d022090 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -775,15 +775,14 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
 static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
                             const ZSTD_CCtx* srcCCtx,
                             ZSTD_frameParameters fParams,
-                            unsigned long long pledgedSrcSize)
+                            unsigned long long pledgedSrcSize,
+                            ZSTD_buffered_policy_e zbuff)
 {
     DEBUGLOG(5, "ZSTD_copyCCtx_internal");
     if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
 
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
-    {   ZSTD_buffered_policy_e const zbuff = srcCCtx->inBuffSize ?
-                        ZSTDb_buffered : ZSTDb_not_buffered;
-        ZSTD_parameters params = srcCCtx->appliedParams;
+    {   ZSTD_parameters params = srcCCtx->appliedParams;
         params.fParams = fParams;
         ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize,
                                 ZSTDcrp_noMemset, zbuff);
@@ -833,9 +832,11 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
 size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize)
 {
     ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
+    ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0);
+    ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1);
     fParams.contentSizeFlag = pledgedSrcSize>0;
 
-    return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize);
+    return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff);
 }
 
 
@@ -3157,6 +3158,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
                                        const void* dict, size_t dictSize,
                                              ZSTD_dictMode_e dictMode)
 {
+    DEBUGLOG(5, "ZSTD_compress_insertDictionary");
     if ((dict==NULL) || (dictSize<=8)) return 0;
 
     /* dict restricted modes */
@@ -3164,8 +3166,10 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
         return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
 
     if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) {
-        if (dictMode == ZSTD_dm_auto)
+        if (dictMode == ZSTD_dm_auto) {
+            DEBUGLOG(5, "raw content dictionary detected");
             return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
+        }
         if (dictMode == ZSTD_dm_fullDict)
             return ERROR(dictionary_wrong);
         assert(0);   /* impossible */
@@ -3184,16 +3188,21 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                                    ZSTD_parameters params, U64 pledgedSrcSize,
                                    ZSTD_buffered_policy_e zbuff)
 {
+    DEBUGLOG(5, "ZSTD_compressBegin_internal");
+    DEBUGLOG(5, "dict ? %s", dict ? "dict" : cdict ? "cdict" : "none");
+    DEBUGLOG(5, "dictMode : %u", (U32)dictMode);
     /* params are supposed to be fully validated at this point */
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
-    if (cdict && cdict->dictContentSize>0)
+    if (cdict && cdict->dictContentSize>0) {
         return ZSTD_copyCCtx_internal(cctx, cdict->refContext,
-                                      params.fParams, pledgedSrcSize);
+                                      params.fParams, pledgedSrcSize,
+                                      zbuff);
+    }
 
-    CHECK_F(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
-                                    ZSTDcrp_continue, zbuff));
+    CHECK_F( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
+                                     ZSTDcrp_continue, zbuff) );
     return ZSTD_compress_insertDictionary(cctx, dict, dictSize, dictMode);
 }
 
@@ -3369,6 +3378,7 @@ static size_t ZSTD_initCDict_internal(
                     unsigned byReference, ZSTD_dictMode_e dictMode,
                     ZSTD_compressionParameters cParams)
 {
+    DEBUGLOG(5, "ZSTD_initCDict_internal, mode %u", (U32)dictMode);
     if ((byReference) || (!dictBuffer) || (!dictSize)) {
         cdict->dictBuffer = NULL;
         cdict->dictContent = dictBuffer;
@@ -3398,7 +3408,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
                                       unsigned byReference, ZSTD_dictMode_e dictMode,
                                       ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
 {
-    DEBUGLOG(5, "ZSTD_createCDict_advanced");
+    DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
     if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
 
     {   ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
@@ -3616,22 +3626,27 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
 }
 
+/*! ZSTD_initCStream_internal() :
+ *  Assumption 1 : params are valid
+ *  Assumption 2 : either dict, or cdict, is defined, not both */
 size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
                     const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
                     ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
+    DEBUGLOG(5, "ZSTD_initCStream_internal");
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
     if (dict && dictSize >= 8) {
+        DEBUGLOG(5, "loading dictionary of size %u", (U32)dictSize);
         if (zcs->staticSize) {   /* static CCtx : never uses malloc */
             /* incompatible with internal cdict creation */
             return ERROR(memory_allocation);
         }
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
-                                                    0 /* byReference */, ZSTD_dm_auto,
-                                                    params.cParams, zcs->customMem);
+                                            0 /* byReference */, ZSTD_dm_auto,
+                                            params.cParams, zcs->customMem);
         zcs->cdict = zcs->cdictLocal;
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
     } else {
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index f07143c1a..8a167ce1d 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -563,11 +563,13 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
                     const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
                     ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
+    DEBUGLOG(5, "ZSTDMT_initCStream_internal");
     /* params are supposed to be fully validated at this point */
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
     if (zcs->nbThreads==1) {
+        DEBUGLOG(5, "single thread mode");
         return ZSTD_initCStream_internal(zcs->cctxPool->cctx[0],
                                 dict, dictSize, cdict,
                                 params, pledgedSrcSize);
@@ -619,6 +621,7 @@ size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
                                 const void* dict, size_t dictSize,
                                 ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
+    DEBUGLOG(5, "ZSTDMT_initCStream_advanced");
     return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, NULL, params, pledgedSrcSize);
 }
 

From 991f9dfcde77df1de6e76af695128ae5368d77e5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 15:16:13 -0700
Subject: [PATCH 100/109] switched fileio.c to ZSTD_DEBUG trigger macro

---
 programs/fileio.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index bbf52168a..d5340cc43 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -106,14 +106,14 @@ static clock_t g_time = 0;
 /*-*************************************
 *  Errors
 ***************************************/
-#ifndef DEBUG
-#  define DEBUG 0
+#ifndef ZSTD_DEBUG
+#  define ZSTD_DEBUG 0
 #endif
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define DEBUGLOG(l,...) if (l<=ZSTD_DEBUG) DISPLAY(__VA_ARGS__);
 #define EXM_THROW(error, ...)                                              \
 {                                                                          \
     DISPLAYLEVEL(1, "zstd: ");                                             \
-    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__);  \
+    DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__);  \
     DISPLAYLEVEL(1, "error %i : ", error);                                 \
     DISPLAYLEVEL(1, __VA_ARGS__);                                          \
     DISPLAYLEVEL(1, " \n");                                                \
@@ -123,7 +123,7 @@ static clock_t g_time = 0;
 #define CHECK(f) {                                   \
     size_t const err = f;                            \
     if (ZSTD_isError(err)) {                         \
-        DEBUGOUTPUT("%s \n", #f);                    \
+        DEBUGLOG(1, "%s \n", #f);                    \
         EXM_THROW(11, "%s", ZSTD_getErrorName(err)); \
 }   }
 

From dce789281ba41e8fdfebd35fa055117c51ace534 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 15:53:42 -0700
Subject: [PATCH 101/109] fixed : decompression of skippable frames in
 streaming mode

---
 lib/decompress/zstd_decompress.c | 22 ++++++++---
 tests/zstreamtest.c              | 63 +++++++++++++++++---------------
 2 files changed, 50 insertions(+), 35 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 4dd78e0e5..c6695679d 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2244,6 +2244,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     char* op = ostart;
     U32 someMoreWork = 1;
 
+    DEBUGLOG(5, "ZSTD_decompressStream");
+    DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
     if (zds->legacyVersion) {
         /* legacy support is incompatible with static dctx */
@@ -2308,12 +2310,20 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             /* Consume header (see ZSTDds_decodeFrameHeader) */
+            DEBUGLOG(5, "Consume header");
             CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
-            CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
-            zds->expected = ZSTD_blockHeaderSize;
-            zds->stage = ZSTDds_decodeBlockHeader;
+
+            if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {  /* skippable frame */
+                zds->expected = MEM_readLE32(zds->headerBuffer + 4);
+                zds->stage = ZSTDds_skipFrame;
+            } else {
+                CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
+                zds->expected = ZSTD_blockHeaderSize;
+                zds->stage = ZSTDds_decodeBlockHeader;
+            }
 
             /* control buffer memory usage */
+            DEBUGLOG(5, "Control max buffer memory usage");
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
             if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
 
@@ -2328,8 +2338,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     DEBUGLOG(5, "outBuff : from %u to %u",
                                 (U32)zds->outBuffSize, (U32)neededOutSize);
                     if (zds->staticSize) {  /* static DCtx */
-                        DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
-                        assert(zds->staticSize >= sizeof(ZSTD_DCtx));  /* already checked at init */
+                        DEBUGLOG(5, "staticSize : %u", (U32)zds->staticSize);
+                        assert(zds->staticSize >= sizeof(ZSTD_DCtx));  /* controlled at init */
                         if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx))
                             return ERROR(memory_allocation);
                     } else {
@@ -2347,7 +2357,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             /* pass-through */
 
         case zdss_read:
+            DEBUGLOG(5, "stage zdss_read");
             {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
+                DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
                 if (neededInSize==0) {  /* end of frame */
                     zds->streamStage = zdss_init;
                     someMoreWork = 0;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 7229f7031..e96e8891c 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -217,9 +217,9 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }
 
     DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
-    { size_t const s = ZSTD_sizeof_CStream(zc);
-      if (ZSTD_isError(s)) goto _output_error;
-      DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
+    {   size_t const s = ZSTD_sizeof_CStream(zc);
+        if (ZSTD_isError(s)) goto _output_error;
+        DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
     }
 
     /* Attempt bad compression parameters */
@@ -234,15 +234,18 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
 
     /* skippable frame test */
     DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
-    ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
+    if (ZSTD_isError( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) ))
+        goto _output_error;
     inBuff.src = compressedBuffer;
     inBuff.size = cSize;
     inBuff.pos = 0;
     outBuff.dst = decodedBuffer;
     outBuff.size = CNBufferSize;
     outBuff.pos = 0;
-    { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
-      if (r != 0) goto _output_error; }
+    {   size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+        DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (U32)r);
+        if (r != 0) goto _output_error;
+    }
     if (outBuff.pos != 0) goto _output_error;   /* skippable frame output len is 0 */
     DISPLAYLEVEL(3, "OK \n");
 
@@ -732,12 +735,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         if ((FUZ_rand(&lseed) & 0xFF) == 131) {
             ZSTD_freeCStream(zc);
             zc = ZSTD_createCStream();
+            CHECK(zc==NULL, "ZSTD_createCStream : allocation error");
             resetAllowed=0;
         }
         if ((FUZ_rand(&lseed) & 0xFF) == 132) {
             ZSTD_freeDStream(zd);
             zd = ZSTD_createDStream();
-            ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
+            CHECK(zd==NULL, "ZSTD_createDStream : allocation error");
+            CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) );  /* ensure at least one init */
         }
 
         /* srcBuffer selection [0-4] */
@@ -829,9 +834,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
 
         /* multi - fragments decompression test */
         if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
-            CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
+            CHECK_Z ( ZSTD_resetDStream(zd) );
         } else {
-            ZSTD_initDStream_usingDict(zd, dict, dictSize);
+            CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
         }
         {   size_t decompressionResult = 1;
             ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
@@ -866,7 +871,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         }   }
 
         /* try decompression on noisy data */
-        ZSTD_initDStream(zd_noise);   /* note : no dictionary */
+        CHECK_Z( ZSTD_initDStream(zd_noise) );   /* note : no dictionary */
         {   ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
             while (outBuff.pos < dstBufferSize) {
@@ -974,11 +979,13 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
             DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads);
             ZSTDMT_freeCCtx(zc);
             zc = ZSTDMT_createCCtx(nbThreads);
+            CHECK(zc==NULL, "ZSTDMT_createCCtx allocation error")
             resetAllowed=0;
         }
         if ((FUZ_rand(&lseed) & 0xFF) == 132) {
             ZSTD_freeDStream(zd);
             zd = ZSTD_createDStream();
+            CHECK(zd==NULL, "ZSTDMT_createCCtx allocation error")
             ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
         }
 
@@ -1003,8 +1010,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
             maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
             if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
             {   int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
-                size_t const resetError = ZSTDMT_initCStream(zc, compressionLevel);
-                CHECK(ZSTD_isError(resetError), "ZSTDMT_initCStream error : %s", ZSTD_getErrorName(resetError));
+                CHECK_Z( ZSTDMT_initCStream(zc, compressionLevel) );
             }
         } else {
             U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
@@ -1028,10 +1034,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
                 params.fParams.contentSizeFlag = pledgedSrcSize>0;
                 DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
-                { size_t const initError = ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize);
-                  CHECK (ZSTD_isError(initError),"ZSTDMT_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); }
-                ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12);
-                ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_sectionSize, FUZ_rand(&lseed) % (2*maxTestSize+1));
+                CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
+                CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12) );
+                CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_sectionSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) );
         }   }
 
         /* multi-segments compression test */
@@ -1049,8 +1054,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                     outBuff.size = outBuff.pos + dstBuffSize;
 
                     DISPLAYLEVEL(5, "Sending %u bytes to compress \n", (U32)srcSize);
-                    { size_t const compressionError = ZSTDMT_compressStream(zc, &outBuff, &inBuff);
-                      CHECK (ZSTD_isError(compressionError), "compression error : %s", ZSTD_getErrorName(compressionError)); }
+                    CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) );
                     DISPLAYLEVEL(5, "%u bytes read by ZSTDMT_compressStream \n", (U32)inBuff.pos);
 
                     XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
@@ -1064,9 +1068,8 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
                     size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
                     outBuff.size = outBuff.pos + adjustedDstSize;
                     DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (U32)adjustedDstSize);
-                    {   size_t const flushError = ZSTDMT_flushStream(zc, &outBuff);
-                        CHECK (ZSTD_isError(flushError), "ZSTDMT_flushStream error : %s", ZSTD_getErrorName(flushError));
-            }   }   }
+                    CHECK_Z( ZSTDMT_flushStream(zc, &outBuff) );
+            }   }
 
             /* final frame epilogue */
             {   size_t remainingToFlush = (size_t)(-1);
@@ -1086,9 +1089,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
 
         /* multi - fragments decompression test */
         if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
-            CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
+            CHECK_Z( ZSTD_resetDStream(zd) );
         } else {
-            ZSTD_initDStream_usingDict(zd, dict, dictSize);
+            CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
         }
         {   size_t decompressionResult = 1;
             ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
@@ -1124,7 +1127,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
         }   }
 
         /* try decompression on noisy data */
-        ZSTD_initDStream(zd_noise);   /* note : no dictionary */
+        CHECK_Z( ZSTD_initDStream(zd_noise) );   /* note : no dictionary */
         {   ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
             while (outBuff.pos < dstBufferSize) {
@@ -1203,7 +1206,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
     RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
     RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
     memset(copyBuffer, 0x65, copyBufferSize);                             /* make copyBuffer considered initialized */
-    ZSTD_initDStream_usingDict(zd, NULL, 0);   /* ensure at least one init */
+    CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) );   /* ensure at least one init */
 
     /* catch up testNb */
     for (testNb=1; testNb < startTest; testNb++)
@@ -1231,13 +1234,13 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
             DISPLAYLEVEL(5, "Creating new context \n");
             ZSTD_freeCCtx(zc);
             zc = ZSTD_createCCtx();
-            CHECK(zc==NULL, "allocation error");
+            CHECK(zc==NULL, "ZSTD_createCCtx allocation error");
             resetAllowed=0;
         }
         if ((FUZ_rand(&lseed) & 0xFF) == 132) {
             ZSTD_freeDStream(zd);
             zd = ZSTD_createDStream();
-            CHECK(zd==NULL, "allocation error");
+            CHECK(zd==NULL, "ZSTD_createDStream allocation error");
             ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
         }
 
@@ -1369,9 +1372,9 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
 
         /* multi - fragments decompression test */
         if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
-            CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
+            CHECK_Z( ZSTD_resetDStream(zd) );
         } else {
-            ZSTD_initDStream_usingDict(zd, dict, dictSize);
+            CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
         }
         {   size_t decompressionResult = 1;
             ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
@@ -1407,7 +1410,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
         }   }
 
         /* try decompression on noisy data */
-        ZSTD_initDStream(zd_noise);   /* note : no dictionary */
+        CHECK_Z( ZSTD_initDStream(zd_noise) );   /* note : no dictionary */
         {   ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
             while (outBuff.pos < dstBufferSize) {

From 224e7a1053cc8c6e118c233105399768bb45b6ab Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 15:58:36 -0700
Subject: [PATCH 102/109] added --list command contrib by @paulcruz74

---
 NEWS | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 3f9b4b220..db9df9634 100644
--- a/NEWS
+++ b/NEWS
@@ -1,10 +1,11 @@
 v1.3.0
-cli : changed : `-t *` does no longer stop after a decompression error
+cli : new : `--list` command, by Paul Cruz
+cli : changed : `-t *` continue processing list after a decompression error
 API : added : ZSTD_versionString()
 API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
 API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
 API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
-API exp : changed : strongest strategy renamed ZSTD_btultra
+API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
 API exp : clarified presentation of memory estimation / measurement functions.
 new : contrib/seekable_format, demo and API, by Sean Purcell
 changed : contrib/linux-kernel, updated version and license, by Nick Terrell

From ecb0f468662c5cdc5aa67ba0bfb59110d9466a1d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 17:25:01 -0700
Subject: [PATCH 103/109] add controls over streaming buffers

---
 lib/compress/zstd_compress.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 06d022090..6b662a9ee 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3743,7 +3743,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
     /* check expectations */
     DEBUGLOG(5, "ZSTD_compressStream_generic");
     assert(zcs->inBuff != NULL);
+    assert(zcs->inBuffSize>0);
     assert(zcs->outBuff!= NULL);
+    assert(zcs->outBuffSize>0);
     assert(output->pos <= output->size);
     assert(input->pos <= input->size);
 

From bfc2f0008066c170fb825b59d4724884a34b19cf Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 17:57:14 -0700
Subject: [PATCH 104/109] --no-big-tests for zstreamtest

Apply --no-big-tests for tsan tests
---
 Makefile            |  3 ++-
 tests/zstreamtest.c | 29 ++++++++++++++++++-----------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index d7a00e431..3cf1cb0b0 100644
--- a/Makefile
+++ b/Makefile
@@ -239,7 +239,8 @@ uasan-%: clean
 	LDFLAGS=-fuse-ld=gold MOREFLAGS="-Og -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*
 
 tsan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
+
 apt-install:
 	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
 
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index e96e8891c..29e48944a 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -689,7 +689,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
     const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
-    int const cLevelLimiter = bigTests ? 3 : 2;
+    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : 17;
 
     /* allocations */
     cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -772,10 +772,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
         } else {
             U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
             U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
-            U32 const cLevel = ( FUZ_rand(&lseed) %
+            U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
                                 (ZSTD_maxCLevel() -
-                                (MAX(testLog, dictLog) / cLevelLimiter)))
+                                (MAX(testLog, dictLog) / 3)))
                                  + 1;
+            U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
             oldTestLog = testLog;
             /* random dictionary selection */
@@ -931,7 +932,8 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
     const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
-    int const cLevelLimiter = bigTests ? 3 : 2;
+    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : 17;
+    U32 const nbThreadsMax = bigTests ? 5 : 2;
 
     /* allocations */
     cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -975,7 +977,8 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
         /* states full reset (deliberately not synchronized) */
         /* some issues can only happen when reusing states */
         if ((FUZ_rand(&lseed) & 0xFF) == 131) {
-            U32 const nbThreads = (FUZ_rand(&lseed) % 6) + 1;
+            U32 const nbThreadsCandidate = (FUZ_rand(&lseed) % 6) + 1;
+            U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax);
             DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads);
             ZSTDMT_freeCCtx(zc);
             zc = ZSTDMT_createCCtx(nbThreads);
@@ -1015,10 +1018,11 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
         } else {
             U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
             U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
-            U32 const cLevel = (FUZ_rand(&lseed) %
+            U32 const cLevelCandidate = (FUZ_rand(&lseed) %
                                (ZSTD_maxCLevel() -
-                               (MAX(testLog, dictLog) / cLevelLimiter))) +
+                               (MAX(testLog, dictLog) / 3))) +
                                1;
+            U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
             oldTestLog = testLog;
             /* random dictionary selection */
@@ -1187,7 +1191,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
     const BYTE* dict = NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
-    int const cLevelLimiter = bigTests ? 3 : 2;
+    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : 17;
+    U32 const nbThreadsMax = bigTests ? 5 : 2;
 
     /* allocations */
     cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -1271,10 +1276,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
         } else {
             U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
             U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
-            U32 const cLevel = (FUZ_rand(&lseed) %
+            U32 const cLevelCandidate = (FUZ_rand(&lseed) %
                                (ZSTD_maxCLevel() -
-                               (MAX(testLog, dictLog) / cLevelLimiter))) +
+                               (MAX(testLog, dictLog) / 3))) +
                                1;
+            U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
             maxTestSize = FUZ_rLogLength(&lseed, testLog);
             oldTestLog = testLog;
             /* random dictionary selection */
@@ -1320,7 +1326,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
 
                 /* multi-threading parameters */
-                {   U32 const nbThreads = (FUZ_rand(&lseed) & 3) + 1;
+                {   U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
+                    U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax);
                     CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, nbThreads) );
                     if (nbThreads > 1) {
                         U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);

From 4d3bdcf13054ffaab6e0298c720f1d48b5882a7a Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 18:20:02 -0700
Subject: [PATCH 105/109] reduced CLevelMax for --no-big-tests

---
 tests/zstreamtest.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 29e48944a..8e58fc872 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -45,6 +45,7 @@
 #define GB *(1U<<30)
 
 static const U32 nbTestsDefault = 10000;
+static const U32 g_cLevelMax_smallTests = 15;
 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
 #define FUZ_COMPRESSIBILITY_DEFAULT 50
 static const U32 prime32 = 2654435761U;
@@ -689,7 +690,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
     const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
-    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : 17;
+    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : g_cLevelMax_smallTests;
 
     /* allocations */
     cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -932,7 +933,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
     const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
-    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : 17;
+    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : g_cLevelMax_smallTests;
     U32 const nbThreadsMax = bigTests ? 5 : 2;
 
     /* allocations */
@@ -1191,7 +1192,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
     const BYTE* dict = NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
-    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : 17;
+    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : g_cLevelMax_smallTests;
     U32 const nbThreadsMax = bigTests ? 5 : 2;
 
     /* allocations */

From 49f8459d11765a0fe462bac69dec033bf3bd47fc Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 18:43:39 -0700
Subject: [PATCH 106/109] fixed minor cast warning

---
 tests/zstreamtest.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 8e58fc872..3ab8d0537 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -690,7 +690,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
     const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
-    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : g_cLevelMax_smallTests;
+    U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
 
     /* allocations */
     cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@@ -933,7 +933,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
     const BYTE* dict=NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
-    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : g_cLevelMax_smallTests;
+    U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
     U32 const nbThreadsMax = bigTests ? 5 : 2;
 
     /* allocations */
@@ -1192,7 +1192,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
     const BYTE* dict = NULL;   /* can keep same dict on 2 consecutive tests */
     size_t dictSize = 0;
     U32 oldTestLog = 0;
-    U32 const cLevelMax = bigTests ? ZSTD_maxCLevel() : g_cLevelMax_smallTests;
+    U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
     U32 const nbThreadsMax = bigTests ? 5 : 2;
 
     /* allocations */

From f99c2c1a229b1a64c77e7e7fd4f15e3d8686aa37 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 21 Jun 2017 23:35:58 -0700
Subject: [PATCH 107/109] reduced --no-big-tests even more

to pass tests on qemu-aarch64
---
 tests/zstreamtest.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 3ab8d0537..c972956c3 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -45,7 +45,7 @@
 #define GB *(1U<<30)
 
 static const U32 nbTestsDefault = 10000;
-static const U32 g_cLevelMax_smallTests = 15;
+static const U32 g_cLevelMax_smallTests = 10;
 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
 #define FUZ_COMPRESSIBILITY_DEFAULT 50
 static const U32 prime32 = 2654435761U;
@@ -670,7 +670,7 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
 
 static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
 {
-    static const U32 maxSrcLog = 24;
+    U32 const maxSrcLog = bigTests ? 24 : 22;
     static const U32 maxSampleLog = 19;
     size_t const srcBufferSize = (size_t)1<
Date: Fri, 23 Jun 2017 00:09:02 -0700
Subject: [PATCH 108/109] zstreamtest : disabled multi-threading tests with
 --newapi --no-big-tests

--no-big-tests is typically used in combination with qemu-user-static
qemu-user-static allocated 4 GB of RAM upfront.
On 2 GB VM, this can degenerate into a crash.
It's not a problem as long as memory is not used.
But with multi-threading enabled, memory fragmentation kicks in,
so the amoung of RAM effectively touched increases,
and can pass beyond the 2 GB limit of the VM.

In single-threaded mode, there is no such issue :
memory requirement is smaller, and remains well-located,
so very little fragmentation is expected.

This modification should make `qemu-arm-static zstreamtests --newapi --no-big-tests`
work fine on Travis CI.
---
 tests/zstreamtest.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 3ab8d0537..33ed11f93 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -1193,7 +1193,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
     size_t dictSize = 0;
     U32 oldTestLog = 0;
     U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
-    U32 const nbThreadsMax = bigTests ? 5 : 2;
+    U32 const nbThreadsMax = bigTests ? 5 : 1;
 
     /* allocations */
     cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);

From 6122af9cf576f7dfd6bb376ca00fdaa7886981f9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 23 Jun 2017 00:37:15 -0700
Subject: [PATCH 109/109] --no-big-tests for msan

msan tests require too much RAM for 2 GB VM
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3cf1cb0b0..679e05625 100644
--- a/Makefile
+++ b/Makefile
@@ -227,7 +227,7 @@ msan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer"   # datagen.c fails this test for no obvious reason
 
 msan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) $*
 
 asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"