modify sequence compression api fuzzer

.gitignore

@@ -26,6 +26,7 @@ tmp*
 *.zstd
 dictionary.
 dictionary
+sequence_fuzz_dictionary
 NUL
 
 # Build artefacts

lib/compress/zstd_compress.c

@@ -288,6 +288,15 @@ static int ZSTD_resolveExternalSequenceValidation(int mode) {
 #endif
 }
 
+/* Resolves maxBlockSize to the default if no value is present. */
+static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
+    if (maxBlockSize == 0) {
+        return ZSTD_BLOCKSIZE_MAX;
+    } else {
+        return maxBlockSize;
+    }
+}
+
 /* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
  * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
 static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
@@ -312,6 +321,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
     cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
     cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
     cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
+    cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
     assert(!ZSTD_checkCParams(cParams));
     return cctxParams;
 }
@@ -377,6 +387,7 @@ ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
     cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
     cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
     cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
+    cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
     DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
                 cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
 }
@@ -604,6 +615,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         bounds.upperBound = 1;
         return bounds;
 
+    case ZSTD_c_maxBlockSize:
+        bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
+        bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
+        return bounds;
+
     default:
         bounds.error = ERROR(parameter_unsupported);
         return bounds;
@@ -670,6 +686,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
     case ZSTD_c_enableMatchFinderFallback:
+    case ZSTD_c_maxBlockSize:
     default:
         return 0;
     }
@@ -727,6 +744,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_deterministicRefPrefix:
     case ZSTD_c_prefetchCDictTables:
     case ZSTD_c_enableMatchFinderFallback:
+    case ZSTD_c_maxBlockSize:
         break;
 
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -964,6 +982,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         CCtxParams->enableMatchFinderFallback = value;
         return CCtxParams->enableMatchFinderFallback;
 
+    case ZSTD_c_maxBlockSize:
+        if (value!=0)    /* 0 ==> default */
+            BOUNDCHECK(ZSTD_c_maxBlockSize, value);
+        CCtxParams->maxBlockSize = value;
+        return CCtxParams->maxBlockSize;
+
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
 }
@@ -1102,6 +1126,9 @@ size_t ZSTD_CCtxParams_getParameter(
     case ZSTD_c_enableMatchFinderFallback:
         *value = CCtxParams->enableMatchFinderFallback;
         break;
+    case ZSTD_c_maxBlockSize:
+        *value = (int)CCtxParams->maxBlockSize;
+        break;
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
     return 0;
@@ -1546,10 +1573,11 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const size_t buffInSize,
         const size_t buffOutSize,
         const U64 pledgedSrcSize,
-        int useExternalMatchFinder)
+        int useExternalMatchFinder,
+        size_t maxBlockSize)
 {
     size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
-    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
+    size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
     size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
@@ -1571,7 +1599,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
 
     size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
     size_t const externalSeqSpace = useExternalMatchFinder
-        ? ZSTD_cwksp_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+        ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
         : 0;
 
     size_t const neededSpace =
@@ -1601,7 +1629,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
      * be needed. However, we still allocate two 0-sized buffers, which can
      * take space under ASAN. */
     return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
-        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
+        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
 }
 
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
@@ -1651,7 +1679,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     {   ZSTD_compressionParameters const cParams =
                 ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
+        size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog);
         size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
                 ? ((size_t)1 << cParams.windowLog) + blockSize
                 : 0;
@@ -1662,7 +1690,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 
         return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
             &cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
-            ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder);
+            ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
     }
 }
 
@@ -1936,6 +1964,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     assert(params->useRowMatchFinder != ZSTD_ps_auto);
     assert(params->useBlockSplitter != ZSTD_ps_auto);
     assert(params->ldmParams.enableLdm != ZSTD_ps_auto);
+    assert(params->maxBlockSize != 0);
     if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
         /* Adjust long distance matching parameters */
         ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, &params->cParams);
@@ -1944,7 +1973,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     }
 
     {   size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
+        size_t const blockSize = MIN(params->maxBlockSize, windowSize);
         size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useExternalMatchFinder);
         size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
                 ? ZSTD_compressBound(blockSize) + 1
@@ -1962,7 +1991,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         size_t const neededSpace =
             ZSTD_estimateCCtxSize_usingCCtxParams_internal(
                 &params->cParams, &params->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
-                buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder);
+                buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder, params->maxBlockSize);
         int resizeWorkspace;
 
         FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
@@ -2340,6 +2369,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
         params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
         params.ldmParams = srcCCtx->appliedParams.ldmParams;
         params.fParams = fParams;
+        params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize;
         ZSTD_resetCCtx_internal(dstCCtx, &params, pledgedSrcSize,
                                 /* loadedDictSize */ 0,
                                 ZSTDcrp_leaveDirty, zbuff);
@@ -3128,7 +3158,6 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
             ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
                                                                                     zc->appliedParams.useRowMatchFinder,
                                                                                     dictMode);
-            assert(zc->externalMatchCtx.mFinder == NULL);
             ms->ldmSeqStore = NULL;
             lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
         }
@@ -4474,7 +4503,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
 {
     ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
     assert(!ZSTD_checkCParams(cParams));
-    return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
+    return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
 }
 
 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
@@ -5912,8 +5941,16 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, &params.cParams);
     params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
     params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
+    params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
 
 #ifdef ZSTD_MULTITHREAD
+    /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
+    RETURN_ERROR_IF(
+        params.useExternalMatchFinder == 1 && params.nbWorkers >= 1,
+        parameter_combination_unsupported,
+        "External matchfinder isn't supported with nbWorkers >= 1"
+    );
+
     if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
         params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
     }
@@ -6103,6 +6140,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
         /* Reset to the original values. */
         cctx->requestedParams.inBufferMode = originalInBufferMode;
         cctx->requestedParams.outBufferMode = originalOutBufferMode;
+
         FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
         if (result != 0) {  /* compression not completed, due to lack of output space */
             assert(oPos == dstCapacity);
@@ -6742,14 +6780,19 @@ void ZSTD_registerExternalMatchFinder(
     ZSTD_CCtx* zc, void* mState,
     ZSTD_externalMatchFinder_F* mFinder
 ) {
-    ZSTD_externalMatchCtx emctx = {
+    if (mFinder != NULL) {
-        mState,
+        ZSTD_externalMatchCtx emctx = {
-        mFinder,
+            mState,
+            mFinder,
 
-        /* seqBuffer is allocated later (from the cwskp) */
+            /* seqBuffer is allocated later (from the cwskp) */
-        NULL, /* seqBuffer */
+            NULL, /* seqBuffer */
-        0 /* seqBufferCapacity */
+            0 /* seqBufferCapacity */
-    };
+        };
-    zc->externalMatchCtx = emctx;
+        zc->externalMatchCtx = emctx;
-    zc->requestedParams.useExternalMatchFinder = 1;
+        zc->requestedParams.useExternalMatchFinder = 1;
+    } else {
+        ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
+        zc->requestedParams.useExternalMatchFinder = 0;
+    }
 }

lib/compress/zstd_compress_internal.h

@@ -355,6 +355,9 @@ struct ZSTD_CCtx_params_s {
      * Users can't set this externally.
      * It is set internally in ZSTD_registerExternalMatchFinder(). */
     int useExternalMatchFinder;
+
+    /* Adjust the max block size*/
+    size_t maxBlockSize;
 };  /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
 
 #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))

lib/zstd.h

@@ -479,6 +479,7 @@ typedef enum {
      * ZSTD_c_useRowMatchFinder
      * ZSTD_c_prefetchCDictTables
      * ZSTD_c_enableMatchFinderFallback
+     * ZSTD_c_maxBlockSize
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -499,7 +500,8 @@ typedef enum {
      ZSTD_c_experimentalParam14=1011,
      ZSTD_c_experimentalParam15=1012,
      ZSTD_c_experimentalParam16=1013,
-     ZSTD_c_experimentalParam17=1014
+     ZSTD_c_experimentalParam17=1014,
+     ZSTD_c_experimentalParam18=1015
 } ZSTD_cParameter;
 
 typedef struct {
@@ -1200,6 +1202,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 #define ZSTD_TARGETLENGTH_MIN     0   /* note : comparing this constant to an unsigned results in a tautological test */
 #define ZSTD_STRATEGY_MIN        ZSTD_fast
 #define ZSTD_STRATEGY_MAX        ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
 
 
 #define ZSTD_OVERLAPLOG_MIN       0
@@ -2112,6 +2115,18 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  * documentation (below) before setting this parameter. */
 #define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17
 
+/*  ZSTD_c_maxBlockSize
+ *  Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ *  The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ *  This parameter can be used to set an upper bound on the blocksize
+ *  that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ *  bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ *  compressBound() innacurate). Only currently meant to be used for testing.
+ *
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
 /*! ZSTD_CCtx_getParameter() :
  *  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
  *  and store it into int* value.
@@ -2825,8 +2840,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* bloc
  * externalMatchState.
  *
  * *** LIMITATIONS ***
- * External matchfinders are compatible with all zstd compression APIs. There are
+ * External matchfinders are compatible with all zstd compression APIs which respect
- * only two limitations.
+ * advanced parameters. However, there are three limitations:
  *
  * First, the ZSTD_c_enableLongDistanceMatching cParam is not supported.
  * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with an
@@ -2848,7 +2863,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* bloc
  *     APIs, work with the external matchfinder, but the external matchfinder won't
  *     receive any history from the previous block. Each block is an independent chunk.
  *
- * Long-term, we plan to overcome both limitations. There is no technical blocker to
+ * Third, multi-threading within a single compression is not supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external matchfinder is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
  * overcoming them. It is purely a question of engineering effort.
  */
 
@@ -2871,6 +2890,17 @@ typedef size_t ZSTD_externalMatchFinder_F (
  * compressions. It will remain set until the user explicitly resets compression
  * parameters.
  *
+ * External matchfinder registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on
+ * compression APIs which respect advanced parameters, such as compress2() and
+ * compressStream(). Older compression APIs such as compressCCtx(), which predate
+ * the introduction of "advanced parameters", will ignore any external matchfinder
+ * setting.
+ *
+ * The external matchfinder can be "cleared" by registering a NULL external
+ * matchfinder function pointer. This removes all limitations described above in
+ * the "LIMITATIONS" section of the API docs.
+ *
  * The user is strongly encouraged to read the full API documentation (above)
  * before calling this function. */
 ZSTDLIB_STATIC_API void

programs/benchzstd.c

@@ -387,12 +387,9 @@ BMK_benchMemAdvancedNoAlloc(
         RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
     }
 
-#if defined(UTIL_TIME_USES_C90_CLOCK)
+    if (!UTIL_support_MT_measurements() && adv->nbWorkers > 1) {
-    if (adv->nbWorkers > 1) {
+        OUTPUTLEVEL(2, "Warning : time measurements may be incorrect in multithreading mode... \n")
-        OUTPUTLEVEL(2, "Warning : time measurements restricted to C90 clock_t. \n")
-        OUTPUTLEVEL(2, "Warning : using C90 clock_t leads to incorrect measurements in multithreading mode. \n")
     }
-#endif
 
     /* Bench */
     {   U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0);

programs/dibio.c

@@ -274,21 +274,20 @@ static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t
     int n;
     memset(&fs, 0, sizeof(fs));
 
-    // We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX
+    /* We assume that if chunking is requested, the chunk size is < SAMPLESIZE_MAX */
     assert( chunkSize <= SAMPLESIZE_MAX );
 
     for (n=0; n<nbFiles; n++) {
       S64 const fileSize = DiB_getFileSize(fileNamesTable[n]);
-      // TODO: is there a minimum sample size? What if the file is 1-byte?
+      /* TODO: is there a minimum sample size? What if the file is 1-byte? */
       if (fileSize == 0) {
         DISPLAYLEVEL(3, "Sample file '%s' has zero size, skipping...\n", fileNamesTable[n]);
         continue;
       }
 
       /* the case where we are breaking up files in sample chunks */
-      if (chunkSize > 0)
+      if (chunkSize > 0) {
-      {
+        /* TODO: is there a minimum sample size? Can we have a 1-byte sample? */
-        // TODO: is there a minimum sample size? Can we have a 1-byte sample?
         fs.nbSamples += (int)((fileSize + chunkSize-1) / chunkSize);
         fs.totalSizeToLoad += fileSize;
       }

programs/fileio_common.h

@@ -122,4 +122,4 @@ extern UTIL_time_t g_displayClock;
 #if defined (__cplusplus)
 }
 #endif
-#endif //ZSTD_FILEIO_COMMON_H
+#endif /* ZSTD_FILEIO_COMMON_H */

programs/timefn.c

@@ -12,7 +12,8 @@
 /* ===  Dependencies  === */
 
 #include "timefn.h"
-
+#include "platform.h" /* set _POSIX_C_SOURCE */
+#include <time.h>     /* CLOCK_MONOTONIC, TIME_UTC */
 
 /*-****************************************
 *  Time functions
@@ -20,12 +21,11 @@
 
 #if defined(_WIN32)   /* Windows */
 
+#include <windows.h>  /* LARGE_INTEGER */
 #include <stdlib.h>   /* abort */
 #include <stdio.h>    /* perror */
 
-UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
+UTIL_time_t UTIL_getTime(void)
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
 {
     static LARGE_INTEGER ticksPerSecond;
     static int init = 0;
@@ -36,16 +36,20 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
         }
         init = 1;
     }
-    return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
+    {   UTIL_time_t r;
+        LARGE_INTEGER x;
+        QueryPerformanceCounter(&x);
+        r.t = (PTime)(x.QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
+        return r;
+    }
 }
 
 
-
 #elif defined(__APPLE__) && defined(__MACH__)
 
-UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
+#include <mach/mach_time.h> /* mach_timebase_info_data_t, mach_timebase_info, mach_absolute_time */
 
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+UTIL_time_t UTIL_getTime(void)
 {
     static mach_timebase_info_data_t rate;
     static int init = 0;
@@ -53,12 +57,39 @@ PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
         mach_timebase_info(&rate);
         init = 1;
     }
-    return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
+    {   UTIL_time_t r;
+        r.t = mach_absolute_time() * (PTime)rate.numer / (PTime)rate.denom;
+        return r;
+    }
+}
+
+/* POSIX.1-2001 (optional) */
+#elif defined(CLOCK_MONOTONIC)
+
+#include <stdlib.h>   /* abort */
+#include <stdio.h>    /* perror */
+
+UTIL_time_t UTIL_getTime(void)
+{
+    /* time must be initialized, othersize it may fail msan test.
+     * No good reason, likely a limitation of timespec_get() for some target */
+    struct timespec time = { 0, 0 };
+    if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) {
+        perror("timefn::clock_gettime(CLOCK_MONOTONIC)");
+        abort();
+    }
+    {   UTIL_time_t r;
+        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
+        return r;
+    }
 }
 
 
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
+/* C11 requires support of timespec_get().
-   Android also lacks it but does define TIME_UTC. */
+ * However, FreeBSD 11 claims C11 compliance while lacking timespec_get().
+ * Double confirm timespec_get() support by checking the definition of TIME_UTC.
+ * However, some versions of Android manage to simultanously define TIME_UTC
+ * and lack timespec_get() support... */
 #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
     && defined(TIME_UTC) && !defined(__ANDROID__)
 
@@ -69,46 +100,38 @@ UTIL_time_t UTIL_getTime(void)
 {
     /* time must be initialized, othersize it may fail msan test.
      * No good reason, likely a limitation of timespec_get() for some target */
-    UTIL_time_t time = UTIL_TIME_INITIALIZER;
+    struct timespec time = { 0, 0 };
     if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
-        perror("timefn::timespec_get");
+        perror("timefn::timespec_get(TIME_UTC)");
         abort();
     }
-    return time;
+    {   UTIL_time_t r;
-}
+        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
-
+        return r;
-static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t diff;
-    if (end.tv_nsec < begin.tv_nsec) {
-        diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
-        diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
-    } else {
-        diff.tv_sec = end.tv_sec - begin.tv_sec;
-        diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
     }
-    return diff;
-}
-
-PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
-{
-    UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
-    PTime nano = 0;
-    nano += 1000000000ULL * diff.tv_sec;
-    nano += diff.tv_nsec;
-    return nano;
 }
 
 
 #else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
 
-UTIL_time_t UTIL_getTime(void) { return clock(); }
+UTIL_time_t UTIL_getTime(void)
-PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
+{
+    UTIL_time_t r;
+    r.t = (PTime)clock() * 1000000000ULL / CLOCKS_PER_SEC;
+    return r;
+}
+
+#define TIME_MT_MEASUREMENTS_NOT_SUPPORTED
 
 #endif
 
 /* ==== Common functions, valid for all time API ==== */
 
+PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
+{
+    return clockEnd.t - clockStart.t;
+}
+
 PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
 {
     return UTIL_getSpanTimeNano(begin, end) / 1000ULL;
@@ -134,3 +157,12 @@ void UTIL_waitForNextTick(void)
         clockEnd = UTIL_getTime();
     } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
 }
+
+int UTIL_support_MT_measurements(void)
+{
+# if defined(TIME_MT_MEASUREMENTS_NOT_SUPPORTED)
+    return 0;
+# else
+    return 1;
+# endif
+}

programs/timefn.h

@@ -18,7 +18,7 @@ extern "C" {
 
 
 /*-****************************************
-*  Local Types
+*  Types
 ******************************************/
 
 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
@@ -32,40 +32,11 @@ extern "C" {
   typedef unsigned long long PTime;  /* does not support compilers without long long support */
 #endif
 
-
+/* UTIL_time_t contains a nanosecond time counter.
-
+ * The absolute value is not meaningful.
-/*-****************************************
+ * It's only valid to compute the difference between 2 measurements. */
-*  Time types (note: OS dependent)
+typedef struct { PTime t; } UTIL_time_t;
-******************************************/
+#define UTIL_TIME_INITIALIZER { 0 }
-#include <time.h>     /* TIME_UTC, then struct timespec and clock_t */
-
-#if defined(_WIN32)   /* Windows */
-
-    #include <windows.h>   /* LARGE_INTEGER */
-    typedef LARGE_INTEGER UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER { { 0, 0 } }
-
-#elif defined(__APPLE__) && defined(__MACH__)
-
-    #include <mach/mach_time.h>
-    typedef PTime UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER 0
-
-/* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance.
-   Android also lacks it but does define TIME_UTC. */
-#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
-    && defined(TIME_UTC) && !defined(__ANDROID__)
-
-    typedef struct timespec UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER { 0, 0 }
-
-#else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
-
-    #define UTIL_TIME_USES_C90_CLOCK
-    typedef clock_t UTIL_time_t;
-    #define UTIL_TIME_INITIALIZER 0
-
-#endif
 
 
 /*-****************************************
@@ -73,16 +44,23 @@ extern "C" {
 ******************************************/
 
 UTIL_time_t UTIL_getTime(void);
+
+/* Timer resolution can be low on some platforms.
+ * To improve accuracy, it's recommended to wait for a new tick
+ * before starting benchmark measurements */
 void UTIL_waitForNextTick(void);
+/* tells if timefn will return correct time measurements
+ * in presence of multi-threaded workload.
+ * note : this is not the case if only C90 clock_t measurements are available */
+int UTIL_support_MT_measurements(void);
 
 PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
 
-#define SEC_TO_MICRO ((PTime)1000000)
 PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
 PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
 
-
+#define SEC_TO_MICRO ((PTime)1000000)  /* nb of microseconds in a second */
 
 
 #if defined (__cplusplus)

tests/decodecorpus.c

@@ -25,21 +25,13 @@
 #include "zdict.h"
 
 /* Direct access to internal compression functions is required */
-#include "zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
+#include "compress/zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
 
 #define XXH_STATIC_LINKING_ONLY
 #include "xxhash.h"     /* XXH64 */
 
-#ifndef MIN
+#if !(defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */))
-    #define MIN(a, b) ((a) < (b) ? (a) : (b))
+# define inline  /* disable */
-#endif
-
-#ifndef MAX_PATH
-    #ifdef PATH_MAX
-        #define MAX_PATH PATH_MAX
-    #else
-        #define MAX_PATH 256
-    #endif
 #endif
 
 /*-************************************
@@ -71,6 +63,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
         }                                                                      \
     } while (0)
 
+
 /*-*******************************************************
 *  Random function
 *********************************************************/
@@ -176,6 +169,14 @@ const char* BLOCK_TYPES[] = {"raw", "rle", "compressed"};
 #define MIN_SEQ_LEN (3)
 #define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)
 
+#ifndef MAX_PATH
+    #ifdef PATH_MAX
+        #define MAX_PATH PATH_MAX
+    #else
+        #define MAX_PATH 256
+    #endif
+#endif
+
 BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
 BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
 BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX];
@@ -241,6 +242,10 @@ typedef enum {
   gt_block,      /* generate compressed blocks without block/frame headers */
 } genType_e;
 
+#ifndef MIN
+    #define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
 /*-*******************************************************
 *  Global variables (set from command line)
 *********************************************************/

tests/external_matchfinder.h

@@ -32,4 +32,4 @@ size_t zstreamExternalMatchFinder(
   size_t windowSize
 );
 
-#endif // EXTERNAL_MATCHFINDER
+#endif /* EXTERNAL_MATCHFINDER */

tests/fuzz/sequence_compression_api.c

@@ -22,6 +22,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <time.h>
+#include <sys/mman.h>
 #include "fuzz_helpers.h"
 #include "zstd_helpers.h"
 #include "fuzz_data_producer.h"
@@ -32,11 +33,17 @@ static void* literalsBuffer = NULL;
 static void* generatedSrc = NULL;
 static ZSTD_Sequence* generatedSequences = NULL;
 
+static void* dictBuffer = NULL;
+static ZSTD_CDict* cdict = NULL;
+static ZSTD_DDict* ddict = NULL;
+
 #define ZSTD_FUZZ_GENERATED_SRC_MAXSIZE (1 << 20) /* Allow up to 1MB generated data */
+#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 20) /* Fixed size 1MB literals buffer */
 #define ZSTD_FUZZ_MATCHLENGTH_MAXSIZE (1 << 18) /* Allow up to 256KB matches */
-#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << 18) /* Allow up to a 256KB dict */
+#define ZSTD_FUZZ_GENERATED_DICT_MAXSIZE (1 << ZSTD_WINDOWLOG_MAX_32) /* Allow up to 1 << ZSTD_WINDOWLOG_MAX_32 dictionary */
-#define ZSTD_FUZZ_GENERATED_LITERALS_SIZE (1 << 18) /* Fixed size 256KB literals buffer */
 #define ZSTD_FUZZ_MAX_NBSEQ (1 << 17) /* Maximum of 128K sequences */
+#define ZSTD_FUZZ_DICT_FILE "sequence_fuzz_dictionary"
+
 
 /* Deterministic random number generator */
 #define FUZZ_RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
@@ -55,9 +62,9 @@ static uint32_t FUZZ_RDG_rand(uint32_t* src)
 /* Make a pseudorandom string - this simple function exists to avoid
  * taking a dependency on datagen.h to have RDG_genBuffer().
  */
-static char* generatePseudoRandomString(char* str, size_t size) {
+static char* generatePseudoRandomString(char* str, size_t size, FUZZ_dataProducer_t* producer) {
     const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
-    uint32_t seed = 0;
+    uint32_t seed = FUZZ_dataProducer_uint32(producer);
     if (size) {
         for (size_t n = 0; n < size; n++) {
             int key = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
@@ -67,6 +74,26 @@ static char* generatePseudoRandomString(char* str, size_t size) {
     return str;
 }
 
+/*
+ * Create large dictionary file
+ */
+static void generateDictFile(size_t size, FUZZ_dataProducer_t* producer) {
+    char c;
+    FILE *dictFile;
+    const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890!@#$^&*()_";
+    uint32_t seed = FUZZ_dataProducer_uint32(producer);
+
+    dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "w");
+    FUZZ_ASSERT(dictFile);
+
+    while (size) {
+      c = FUZZ_RDG_rand(&seed) % (int) (sizeof charset - 1);
+      fputc(c, dictFile);
+      size--;
+    }
+    fclose(dictFile);
+}
+
 /* Returns size of source buffer */
 static size_t decodeSequences(void* dst, size_t nbSequences,
                               size_t literalsSize,
@@ -100,14 +127,14 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
             size_t j = 0;
             size_t k = 0;
             if (dictSize != 0) {
-                if (generatedSequences[i].offset > bytesWritten) {
+                if (generatedSequences[i].offset > bytesWritten) { /* Offset goes into the dictionary */
-                    /* Offset goes into the dictionary */
+                    size_t dictOffset = generatedSequences[i].offset - bytesWritten;
-                    size_t offsetFromEndOfDict = generatedSequences[i].offset - bytesWritten;
+                    size_t matchInDict = MIN(matchLength, dictOffset);
-                    for (; k < offsetFromEndOfDict && k < matchLength; ++k) {
+                    for (; k < matchInDict; ++k) {
-                        op[k] = dictPtr[dictSize - offsetFromEndOfDict + k];
+                        op[k] = dictPtr[dictSize - dictOffset + k];
                     }
-                    matchLength -= k;
+                    matchLength -= matchInDict;
-                    op += k;
+                    op += matchInDict;
                 }
             }
             for (; j < matchLength; ++j) {
@@ -138,9 +165,9 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
                                       size_t literalsSizeLimit, size_t dictSize,
                                       size_t windowLog, ZSTD_sequenceFormat_e mode)
 {
-    const uint32_t repCode = 0;  /* not used by sequence ingestion api */
+    const uint32_t repCode = 0;  /* Not used by sequence ingestion api */
-    const uint32_t windowSize = 1 << windowLog;
+    size_t windowSize = 1ULL << windowLog;
-    const uint32_t blockSizeMax = MIN(128 << 10, 1 << windowLog);
+    size_t blockSizeMax = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
     uint32_t matchLengthMax = ZSTD_FUZZ_MATCHLENGTH_MAXSIZE;
     uint32_t bytesGenerated = 0;
     uint32_t nbSeqGenerated = 0;
@@ -148,12 +175,12 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
     uint32_t blockSize = 0;
 
     if (mode == ZSTD_sf_explicitBlockDelimiters) {
-        /* ensure that no sequence can be larger than one block */
+        /* Ensure that no sequence can be larger than one block */
         literalsSizeLimit = MIN(literalsSizeLimit, blockSizeMax/2);
         matchLengthMax = MIN(matchLengthMax, blockSizeMax/2);
     }
 
-    while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* extra room for explicit delimiters */
+    while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* Extra room for explicit delimiters */
          && bytesGenerated < ZSTD_FUZZ_GENERATED_SRC_MAXSIZE
          && !FUZZ_dataProducer_empty(producer)) {
         uint32_t matchLength;
@@ -210,38 +237,31 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
             }
             generatedSequences[nbSeqGenerated++] = seq;
             isFirstSequence = 0;
-    }   }
+        }
+    }
+
     if (mode == ZSTD_sf_explicitBlockDelimiters) {
         /* always end sequences with a block delimiter */
         const ZSTD_Sequence endBlock = {0, 0, 0, 0};
         assert(nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ);
         generatedSequences[nbSeqGenerated++] = endBlock;
     }
-
     return nbSeqGenerated;
 }
 
 static size_t roundTripTest(void* result, size_t resultCapacity,
                             void* compressed, size_t compressedCapacity,
                             const void* src, size_t srcSize,
-                            const void* dict, size_t dictSize,
                             const ZSTD_Sequence* seqs, size_t seqSize,
-                            int wLog, int cLevel, unsigned hasDict,
+                            unsigned hasDict,
                             ZSTD_sequenceFormat_e mode)
 {
     size_t cSize;
     size_t dSize;
 
-    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
-    ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
     if (hasDict) {
-        FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary(cctx, dict, dictSize));
+        FUZZ_ZASSERT(ZSTD_CCtx_refCDict(cctx, cdict));
-        FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary(dctx, dict, dictSize));
+        FUZZ_ZASSERT(ZSTD_DCtx_refDDict(dctx, ddict));
     }
 
     cSize = ZSTD_compressSequences(cctx, compressed, compressedCapacity,
@@ -272,7 +292,6 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     size_t cBufSize;
     size_t generatedSrcSize;
     size_t nbSequences;
-    void* dictBuffer = NULL;
     size_t dictSize = 0;
     unsigned hasDict;
     unsigned wLog;
@@ -281,23 +300,66 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
 
     FUZZ_dataProducer_t* const producer = FUZZ_dataProducer_create(src, size);
     FUZZ_ASSERT(producer);
-    if (literalsBuffer == NULL) {
+
+    if (!cctx) {
+        cctx = ZSTD_createCCtx();
+        FUZZ_ASSERT(cctx);
+    }
+    if (!dctx) {
+        dctx = ZSTD_createDCtx();
+        FUZZ_ASSERT(dctx);
+    }
+
+    /* Generate window log first so we don't generate offsets too large */
+    wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
+    cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
+    mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
+
+    ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach);
+
+    if (!literalsBuffer) {
         literalsBuffer = FUZZ_malloc(ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
         FUZZ_ASSERT(literalsBuffer);
-        literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE);
+        literalsBuffer = generatePseudoRandomString(literalsBuffer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, producer);
     }
 
+    if (!dictBuffer) { /* Generate global dictionary buffer */
+        FILE* dictFile;
+        ZSTD_compressionParameters cParams;
+
+        /* Generate a large dictionary file and mmap to buffer */
+        generateDictFile(ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, producer);
+        dictFile = fopen(ZSTD_FUZZ_DICT_FILE, "r");
+        dictBuffer = mmap(NULL, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, PROT_READ, MAP_PRIVATE, fileno(dictFile), 0);
+        FUZZ_ASSERT(dictBuffer);
+        fclose(dictFile);
+
+        /* Create global cdict and ddict*/
+        cParams = ZSTD_getCParams(1, ZSTD_FUZZ_GENERATED_SRC_MAXSIZE, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
+        cParams.minMatch = ZSTD_MINMATCH_MIN;
+        cParams.hashLog = ZSTD_HASHLOG_MIN;
+        cParams.chainLog = ZSTD_CHAINLOG_MIN;
+
+        cdict = ZSTD_createCDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
+        ddict = ZSTD_createDDict_advanced(dictBuffer, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
+        FUZZ_ASSERT(cdict);
+        FUZZ_ASSERT(ddict);
+    }
+
+    FUZZ_ASSERT(cdict);
+    FUZZ_ASSERT(ddict);
+
     hasDict = FUZZ_dataProducer_uint32Range(producer, 0, 1);
     if (hasDict) {
-        dictSize = FUZZ_dataProducer_uint32Range(producer, 1, ZSTD_FUZZ_GENERATED_DICT_MAXSIZE);
+        dictSize = ZSTD_FUZZ_GENERATED_DICT_MAXSIZE;
-        dictBuffer = FUZZ_malloc(dictSize);
-        FUZZ_ASSERT(dictBuffer);
-        dictBuffer = generatePseudoRandomString(dictBuffer, dictSize);
     }
-    /* Generate window log first so we don't generate offsets too large */
-    wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX_32);
-    cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
-    mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
 
     if (!generatedSequences) {
         generatedSequences = FUZZ_malloc(sizeof(ZSTD_Sequence)*ZSTD_FUZZ_MAX_NBSEQ);
@@ -305,8 +367,10 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     if (!generatedSrc) {
         generatedSrc = FUZZ_malloc(ZSTD_FUZZ_GENERATED_SRC_MAXSIZE);
     }
+
     nbSequences = generateRandomSequences(producer, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictSize, wLog, mode);
     generatedSrcSize = decodeSequences(generatedSrc, nbSequences, ZSTD_FUZZ_GENERATED_LITERALS_SIZE, dictBuffer, dictSize, mode);
+
     /* Note : in explicit block delimiters mode,
      * the fuzzer might generate a lot of small blocks.
      * In which case, the final compressed size might be > ZSTD_compressBound().
@@ -318,30 +382,17 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
     rBufSize = generatedSrcSize;
     rBuf = FUZZ_malloc(rBufSize);
 
-    if (!cctx) {
-        cctx = ZSTD_createCCtx();
-        FUZZ_ASSERT(cctx);
-    }
-    if (!dctx) {
-        dctx = ZSTD_createDCtx();
-        FUZZ_ASSERT(dctx);
-    }
-
     {   const size_t result = roundTripTest(rBuf, rBufSize,
                                         cBuf, cBufSize,
                                         generatedSrc, generatedSrcSize,
-                                        dictBuffer, dictSize,
                                         generatedSequences, nbSequences,
-                                        (int)wLog, cLevel, hasDict, mode);
+                                        hasDict, mode);
         FUZZ_ASSERT(result <= generatedSrcSize);  /* can be 0 when no round-trip */
     }
 
     free(rBuf);
     free(cBuf);
     FUZZ_dataProducer_free(producer);
-    if (hasDict) {
-        free(dictBuffer);
-    }
 #ifndef STATEFUL_FUZZING
     ZSTD_freeCCtx(cctx); cctx = NULL;
     ZSTD_freeDCtx(dctx); dctx = NULL;

tests/fuzz/zstd_helpers.c

@@ -115,6 +115,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
     setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer);
     setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
     setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
+    setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer);
     if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
       setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
     }

tests/fuzzer.c

@@ -25,7 +25,8 @@
 #include <stdlib.h>       /* free */
 #include <stdio.h>        /* fgets, sscanf */
 #include <string.h>       /* strcmp */
-#undef NDEBUG
+#include <time.h>         /* time(), time_t */
+#undef NDEBUG             /* always enable assert() */
 #include <assert.h>
 #define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_compressContinue, ZSTD_compressBlock */
 #include "debug.h"        /* DEBUG_STATIC_ASSERT */
@@ -476,7 +477,7 @@ static void test_compressBound(unsigned tnb)
             CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
     }   }
 
-    // Ensure error if srcSize too big
+    /* Ensure error if srcSize too big */
     {   size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
         CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
         CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
@@ -489,7 +490,7 @@ static void test_decompressBound(unsigned tnb)
 {
     DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
 
-    // Simple compression, with size : should provide size;
+    /* Simple compression, with size : should provide size; */
     {   const char example[] = "abcd";
         char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
         size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
@@ -497,7 +498,7 @@ static void test_decompressBound(unsigned tnb)
         CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
     }
 
-    // Simple small compression without size : should provide 1 block size
+    /* Simple small compression without size : should provide 1 block size */
     {   char cBuffer[ZSTD_COMPRESSBOUND(0)];
         ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
         ZSTD_inBuffer in = { NULL, 0, 0 };
@@ -510,14 +511,14 @@ static void test_decompressBound(unsigned tnb)
         ZSTD_freeCCtx(cctx);
     }
 
-    // Attempt to overflow 32-bit intermediate multiplication result
+    /* Attempt to overflow 32-bit intermediate multiplication result
-    // This requires dBound >= 4 GB, aka 2^32.
+     * This requires dBound >= 4 GB, aka 2^32.
-    // This requires 2^32 / 2^17 = 2^15 blocks
+     * This requires 2^32 / 2^17 = 2^15 blocks
-    // => create 2^15 blocks (can be empty, or just 1 byte).
+     * => create 2^15 blocks (can be empty, or just 1 byte). */
     {   const char input[] = "a";
         size_t const nbBlocks = (1 << 15) + 1;
         size_t blockNb;
-        size_t const outCapacity = 1 << 18; // large margin
+        size_t const outCapacity = 1 << 18; /* large margin */
         char* const outBuffer = malloc (outCapacity);
         ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
         ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -1779,6 +1780,94 @@ static int basicUnitTests(U32 const seed, double compressibility)
                 if (!ZSTD_isError(r)) goto _output_error;
             }
             DISPLAYLEVEL(3, "OK \n");
+
+            DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
+            {
+                // Test ZSTD_estimateCCtxSize_usingCCtxParams
+                {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3));
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+
+                // Test ZSTD_estimateCStreamSize_usingCCtxParams
+                  {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3) );
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+            }
+            DISPLAYLEVEL(3, "OK \n");
+
+            DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
+            {
+                // Test ZSTD_estimateCCtxSize_usingCCtxParams
+                {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t cctxSizeDefault;
+                    CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
+                    cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3) );
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+
+                // Test ZSTD_estimateCStreamSize_usingCCtxParams
+                  {
+                    ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+                    size_t cctxSizeDefault;
+                    CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
+                    cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
+                    staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
+                    CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
+                                    compressedBuffer, compressedBufferSize,
+                                    CNBuffer, CNBuffSize, 3) );
+
+                    {
+                        size_t const r = ZSTD_decompressDCtx(staticDCtx,
+                                                    decodedBuffer, CNBuffSize,
+                                                    compressedBuffer, cSize);
+                                                                        if (r != CNBuffSize) goto _output_error;
+                        if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
+                    }
+                    ZSTD_freeCCtxParams(params);
+                }
+            }
+            DISPLAYLEVEL(3, "OK \n");
         }
         free(staticCCtxBuffer);
         free(staticDCtxBuffer);
@@ -3535,7 +3624,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
 
     DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
     {
-        U32 seed_copy = seed; // need non-const seed to avoid compiler warning for FUZ_rand(&seed)
+        U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
         U32 rand32 = FUZ_rand(&seed_copy);
         U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
         U32 lowbit_only_32 = 1;
@@ -3543,8 +3632,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
         U32 highbit_only_32 = (U32)1 << 31;
         U64 highbit_only_64 = (U64)1 << 63;
         U32 i;
-        if (rand32 == 0) rand32 = 1; // CLZ and CTZ are undefined on 0
+        if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
-        if (rand64 == 0) rand64 = 1; // CLZ and CTZ are undefined on 0
+        if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
 
         /* Test ZSTD_countTrailingZeros32 */
         CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);

tests/test-zstd-versions.py

@@ -29,8 +29,19 @@ test_dat_src = 'README.md'
 test_dat = 'test_dat'
 head = 'vdevel'
 dict_source = 'dict_source'
-dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
+dict_globs = [
-dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
+    'programs/*.c',
+    'lib/common/*.c',
+    'lib/compress/*.c',
+    'lib/decompress/*.c',
+    'lib/dictBuilder/*.c',
+    'lib/legacy/*.c',
+    'programs/*.h',
+    'lib/common/*.h',
+    'lib/compress/*.h',
+    'lib/dictBuilder/*.h',
+    'lib/legacy/*.h'
+]
 
 
 def execute(command, print_output=False, print_error=True, param_shell=False):
@@ -85,6 +96,7 @@ def create_dict(tag, dict_source_path):
             result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
         if result == 0:
             print(dict_name + ' created')
+            assert os.path.isfile(dict_name)
         else:
             raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
     else:
@@ -103,12 +115,15 @@ def zstd(tag, args, input_file, output_file):
             print("Running: '{}', input={}, output={}" .format(
                 ' '.join(cmd), input_file, output_file
             ))
-            subprocess.check_call(cmd, stdin=i, stdout=o)
+            result = subprocess.run(cmd, stdin=i, stdout=o, stderr=subprocess.PIPE)
+            print("Stderr: {}".format(result.stderr.decode("ascii")))
+            result.check_returncode()
 
 
 def dict_compress_sample(tag, sample):
     dict_name = 'dict.' + tag
-    zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst')
+    verbose = ['-v', '-v', '-v']
+    zstd(tag, ['-D', dict_name, '-1'] + verbose, sample, sample + '_01_64_' + tag + '_dictio.zst')
     zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
     zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
     zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
@@ -246,8 +261,12 @@ if __name__ == '__main__':
     # copy *.c and *.h to a temporary directory ("dict_source")
     if not os.path.isdir(dict_source_path):
         os.mkdir(dict_source_path)
-        print('cp ' + dict_files + ' ' + dict_source_path)
+        for dict_glob in dict_globs:
-        execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
+            files = glob.glob(dict_glob, root_dir=base_dir)
+            for file in files:
+                file = os.path.join(base_dir, file)
+                print("copying " + file + " to " + dict_source_path)
+                shutil.copy(file, dict_source_path)
 
     print('-----------------------------------------------')
     print('Compress test.dat by all released zstd')

tests/zstreamtest.c

@@ -1846,15 +1846,11 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
 
         CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
 
-        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
 
         /* Reference external matchfinder outside the test loop to
          * check that the reference is preserved across compressions */
-        ZSTD_registerExternalMatchFinder(
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
-            zc,
-            &externalMatchState,
-            zstreamExternalMatchFinder
-        );
 
         for (enableFallback = 0; enableFallback < 1; enableFallback++) {
             size_t testCaseId;
@@ -1916,16 +1912,160 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
         }
 
         /* Test that reset clears the external matchfinder */
-        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
         externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
         CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
         CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
 
+        /* Test that registering mFinder == NULL clears the external matchfinder */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        ZSTD_registerExternalMatchFinder(zc, NULL, NULL); /* clear the external matchfinder */
+        CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
+
+        /* Test that external matchfinder doesn't interact with older APIs */
+        ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
+        ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+        externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
+        CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
+        CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
+
+        /* Test that compression returns the correct error with LDM */
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        {
+            size_t res;
+            ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
+            res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+            CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+            CHECK(
+                ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+                "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+            );
+        }
+
+#ifdef ZSTD_MULTITHREAD
+        /* Test that compression returns the correct error with nbWorkers > 0 */
+        CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
+        {
+            size_t res;
+            ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
+            CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
+            res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
+            CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
+            CHECK(
+                ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
+                "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
+            );
+        }
+#endif
+
         free(dstBuf);
         free(checkBuf);
     }
     DISPLAYLEVEL(3, "OK \n");
 
+
+    /* Test maxBlockSize cctx param functionality */
+    DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
+    {
+        ZSTD_CCtx* cctx = ZSTD_createCCtx();
+
+        /* Quick test to make sure maxBlockSize bounds are enforced */
+        assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
+        assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
+
+        /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
+        {
+            size_t srcSize = 2 << 10;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst1 = compressedBuffer;
+            void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+            size_t size1, size2;
+            void* const checkBuf = malloc(srcSize);
+            memset(src, 'x', srcSize);
+
+            /* maxBlockSize = 1KB */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = 3KB */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 - size2 == 4); /* We add another RLE block with header + character */
+            assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
+
+            /* maxBlockSize = 1KB, windowLog = 10 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = 3KB, windowLog = 10 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 == size2);
+            assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+
+            free(checkBuf);
+        }
+
+        ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
+
+        /* Test maxBlockSize = 0 is valid */
+        {   size_t srcSize = 256 << 10;
+            void* const src = CNBuffer;
+            size_t dstSize = ZSTD_compressBound(srcSize);
+            void* const dst1 = compressedBuffer;
+            void* const dst2 = (BYTE*)compressedBuffer + dstSize;
+            size_t size1, size2;
+            void* const checkBuf = malloc(srcSize);
+
+            /* maxBlockSize = 0 */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
+            size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size1)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
+            CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
+            size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
+
+            if (ZSTD_isError(size2)) goto _output_error;
+            CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
+            CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
+
+            assert(size1 == size2);
+            assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
+            free(checkBuf);
+        }
+        ZSTD_freeCCtx(cctx);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
 _end:
     FUZ_freeDictionary(dictionary);
     ZSTD_freeCStream(zc);