Reduce Duplication

Change Defaults
Asserts actually disabled in paramgrill + fullbench

tests/Makefile

@@ -129,7 +129,7 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
 fullbench32: CPPFLAGS += -m32
 fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
 fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
-fullbench fullbench32 : DEBUGFLAGS =   # turn off assert() for speed measurements
+fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG  # turn off assert() for speed measurements
 fullbench fullbench32 : $(ZSTD_FILES)
 fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
 	$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c  # xxh symbols not exposed from dll
 zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 
-paramgrill : DEBUGFLAGS =   # turn off assert() for speed measurements
+paramgrill : DEBUGFLAGS = -DNDEBUG  # turn off assert() for speed measurements
 paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c
 	$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
 

tests/README.md

@@ -118,8 +118,8 @@ Full list of arguments
                 (Lower value will begin with stronger strategies) (Default 90%)
     preferSpeed= / preferRatio=
               : Only affects lvl = invocations. Defines value placed on compression speed or ratio
-                when determining overall winner (default 1 for both, higher = more valued).
+                when determining overall winner (default speed = 1, ratio = 5 for both, higher = more valued).
-    tries=    : Maximum number of random restarts on a single strategy before switching (Default 5)
+    tries=    : Maximum number of random restarts on a single strategy before switching (Default 3)
                 Higher values will make optimizer run longer, more chances to find better solution.
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 

tests/paramgrill.c

@@ -139,7 +139,7 @@ static BMK_result_t g_lvltarget;
 static int g_optmode = 0;
 
 static U32 g_speedMultiplier = 1;
-static U32 g_ratioMultiplier = 1;
+static U32 g_ratioMultiplier = 5;
 
 /* g_mode? */
 
@@ -382,7 +382,7 @@ typedef struct {
 } contexts_t;
 
 /*-*******************************************************
-*  From Paramgrill
+*  From bench.c
 *********************************************************/
 
 static void BMK_initCCtx(ZSTD_CCtx* ctx, 
@@ -411,7 +411,6 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
     ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
 }
 
-
 static void BMK_initDCtx(ZSTD_DCtx* dctx, 
     const void* dictBuffer, const size_t dictBufferSize) {
     ZSTD_DCtx_reset(dctx);
@@ -503,7 +502,7 @@ static size_t local_defaultDecompress(
 }
 
 /*-*******************************************************
-*  From Paramgrill End
+*  From bench.c End
 *********************************************************/
 
 static void freeBuffers(const buffers_t b) {
@@ -527,23 +526,25 @@ static void freeBuffers(const buffers_t b) {
     free(b.resSizes);
 }
 
-/* allocates buffer's arguments. returns success / failuere */
+/* srcBuffer will be freed by freeBuffers now */
-static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, 
+static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles,
-                          const size_t nbFiles)
+    const size_t* fileSizes)
 {
-    size_t pos = 0;
+    size_t pos = 0, n, blockSize;
-    size_t n;
+    U32 maxNbBlocks, blockNb = 0;
-    U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
+    buff->srcSize = 0;
-    const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
+    for(n = 0; n < nbFiles; n++) {
-    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
+        buff->srcSize += fileSizes[n];
-    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / MAX(blockSize, 1)) + (U32)nbFiles;
+    }
-    U32 blockNb = 0;
 
-    if(!totalSizeToLoad || !benchedSize) {
+    if(buff->srcSize == 0) {
-        DISPLAY("Nothing to Bench\n");
+        DISPLAY("No data to bench\n");
         return 1;
     }
-    
+
+    blockSize = g_blockSize ? g_blockSize : buff->srcSize;
+    maxNbBlocks = (U32) ((buff->srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles;
+
     buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
     buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
 
@@ -560,18 +561,62 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         return 1;
     }
 
-
+    buff->srcBuffer = srcBuffer;
-    buff->srcBuffer = malloc(benchedSize);
     buff->srcPtrs[0] = (const void*)buff->srcBuffer;
-    buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
+    buff->dstPtrs[0] = malloc(ZSTD_compressBound(buff->srcSize) + (maxNbBlocks * 1024));
-    buff->resPtrs[0] = malloc(benchedSize);
+    buff->resPtrs[0] = malloc(buff->srcSize);
 
-    if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) {
+    if(!buff->dstPtrs[0] || !buff->resPtrs[0]) {
         DISPLAY("alloc error\n");
         freeBuffers(*buff);
         return 1;
     }
 
+    for(n = 0; n < nbFiles; n++) {
+        size_t pos_end = pos + fileSizes[n];
+        for(; pos < pos_end; blockNb++) {
+            buff->srcPtrs[blockNb] = (const void*)((char*)srcBuffer + pos);
+            buff->srcSizes[blockNb] = blockSize;
+            pos += blockSize;
+        }
+
+        if(fileSizes[n] > 0) { buff->srcSizes[blockNb - 1] = ((fileSizes[n] - 1) % blockSize) + 1; }
+        pos = pos_end;
+    }
+
+    buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
+    buff->dstSizes[0] = buff->dstCapacities[0];
+    buff->resSizes[0] = buff->srcSizes[0];
+
+    for(n = 1; n < blockNb; n++) {
+        buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
+        buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
+        buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
+        buff->dstSizes[n] = buff->dstCapacities[n];
+        buff->resSizes[n] = buff->srcSizes[n];
+    }
+
+    buff->nbBlocks = blockNb;
+
+    return 0;
+} 
+
+/* allocates buffer's arguments. returns success / failuere */
+static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, 
+                          size_t nbFiles) {
+    size_t pos = 0;
+    size_t n;
+    size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
+    size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
+    size_t* fileSizes = calloc(sizeof(size_t), nbFiles);
+    void* srcBuffer = malloc(benchedSize);
+    int ret = 0;  
+
+
+    if(!fileSizes || !srcBuffer) {
+        return 1;
+    }
+
     for(n = 0; n < nbFiles; n++) {
         FILE* f;
         U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
@@ -586,62 +631,35 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         f = fopen(fileNamesTable[n], "rb");
         if (f==NULL) {
             DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
-            freeBuffers(*buff);
+            free(fileSizes);
+            free(srcBuffer);
             fclose(f);
             return 10;
         }
 
         DISPLAY("Loading %s...       \r", fileNamesTable[n]);
 
-        if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n=nbFiles;   /* buffer too small - stop after this file */
+        if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n;   /* buffer too small - stop after this file */
         {
-            char* buffer = (char*)(buff->srcBuffer); 
+            char* buffer = (char*)(srcBuffer); 
             size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
-            size_t blocked = 0;
-            while(blocked < readSize) {
-                buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked));
-                buff->srcSizes[blockNb] = blockSize;
-                blocked += blockSize;
-                blockNb++;
-            }
-            if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; }
 
-            if (readSize != (size_t)fileSize) {
+            if (readSize != (size_t)fileSize) { /* should we accept partial read? */
                 DISPLAY("could not read %s", fileNamesTable[n]);
-                freeBuffers(*buff);
+                free(fileSizes);
-                fclose(f);
+                free(srcBuffer);
                 return 1;
             }
 
+            fileSizes[n] = readSize;
             pos += readSize;
-
         }
         fclose(f);
     }
 
-    if(!blockNb) {
+    ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes);
-        DISPLAY("Failed to load any files\n");
+    free(fileSizes);
-        freeBuffers(*buff);
+    return ret;
-        return 1;
-    }
-
-    buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
-    buff->dstSizes[0] = buff->dstCapacities[0];
-    buff->resSizes[0] = buff->srcSizes[0];
-
-    for(n = 1; n < blockNb; n++) {
-        buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
-        buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
-        buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
-        buff->dstSizes[n] = buff->dstCapacities[n];
-        buff->resSizes[n] = buff->srcSizes[n];
-    }
-    buff->srcSize = pos;
-    buff->nbBlocks = blockNb;
-
-    if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; }
-
-    return 0;
 }
 
 static void freeContexts(const contexts_t ctx) {
@@ -1611,7 +1629,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
         BMK_init_level_constraints(g_target * (1 MB));
     } else {
         /* baseline config for level 1 */
-        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? 
+        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize);
         BMK_result_t testResult;
         BMK_benchParam(&testResult, buf, ctx, l1params);
         BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
@@ -1641,82 +1659,6 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl
     fclose(f);
 }
 
-static int benchSample(void)
-{
-    const char* const name = "Sample 10MB";
-    size_t const benchedSize = 10 MB;
-    U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
-    U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1;
-    size_t splitSize = 0;
-
-    buffers_t buf;
-    contexts_t ctx;
-
-    buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
-    buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
-    buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
-    buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buf.srcSize = benchedSize;
-
-    if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) {
-        DISPLAY("Allocation Error\n");
-        freeBuffers(buf);
-        return 1;
-    } 
-
-    buf.srcBuffer = malloc(benchedSize);
-    buf.srcPtrs[0] = (const void*)buf.srcBuffer;
-    buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks);
-    buf.resPtrs[0] = malloc(benchedSize);
-
-    if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) {
-        DISPLAY("Allocation Error\n");
-        freeBuffers(buf);
-        return 1;
-    }
-
-
-    splitSize = MIN(benchedSize, blockSize);
-    buf.srcSizes[0] = splitSize;
-    buf.dstCapacities[0] = ZSTD_compressBound(splitSize);
-    buf.resSizes[0] = splitSize;
-
-    for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) {
-        const size_t i = buf.nbBlocks;
-        const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize);
-        buf.srcSizes[i] = nextBlockSize;
-        buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize);
-        buf.resSizes[i] = nextBlockSize;
-        buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]);
-        buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]);
-        buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]);
-        splitSize += nextBlockSize;
-    }
-
-    if(createContexts(&ctx, NULL)) {
-        DISPLAY("Context Creation Error\n");
-        freeBuffers(buf);
-        return 1;
-    }
-
-    RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0);
-
-    /* bench */
-    DISPLAY("\r%79s\r", "");
-    DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
-
-    BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
-
-    freeBuffers(buf);
-    freeContexts(ctx);
-
-    return 0;
-}
-
-
 static int benchOnce(buffers_t buf, contexts_t ctx) {
     BMK_result_t testResult;
 
@@ -1726,9 +1668,56 @@ static int benchOnce(buffers_t buf, contexts_t ctx) {
     }
 
     BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize);
+
     return 0;
 }
 
+static int benchSample(void)
+{
+    const char* const name = "Sample 10MB";
+    size_t const benchedSize = 10 MB;
+    U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
+    void* srcBuffer = malloc(benchedSize);
+    int ret = 0;
+
+    buffers_t buf;
+    contexts_t ctx;
+    
+    if(srcBuffer == NULL) {
+        DISPLAY("Out of Memory\n");
+        return 2;
+    }
+
+    RDG_genBuffer(srcBuffer, benchedSize, g_compressibility, 0.0, 0);
+
+    if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) {
+        DISPLAY("Buffer Creation Error\n");
+        free(srcBuffer);
+        return 3;
+    }
+
+    if(createContexts(&ctx, NULL)) {
+        DISPLAY("Context Creation Error\n");
+        freeBuffers(buf);
+        return 1;
+    }
+
+    /* bench */
+    DISPLAY("\r%79s\r", "");
+    DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
+
+    if(g_singleRun) {
+        ret = benchOnce(buf, ctx);
+    } else {
+        BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
+    }
+
+    freeBuffers(buf);
+    freeContexts(ctx);
+
+    return ret;
+}
+
 /* benchFiles() :
  * note: while this function takes a table of filenames,
  * in practice, only the first filename will be used */
@@ -1776,6 +1765,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
 
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
 #define VARIANCE 1.2 
+#define HIGH_VARIANCE 100.0
 static int allBench(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx,
                 const ZSTD_compressionParameters cParams,
@@ -1784,7 +1774,7 @@ static int allBench(BMK_result_t* resultPtr,
     BMK_return_t benchres;
     BMK_result_t resultMax;
     U64 loopDurationC = 0, loopDurationD = 0;
-    double uncertaintyConstantC, uncertaintyConstantD;
+    double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
     double winnerRS;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
@@ -1802,18 +1792,12 @@ static int allBench(BMK_result_t* resultPtr,
     /* calculate uncertainty in compression / decompression runs */
     if(benchres.result.cSpeed) {
         loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); 
-        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; 
+        uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); 
-    } else {
-        loopDurationC = 0;
-        uncertaintyConstantC = 3;
     }
 
     if(benchres.result.dSpeed) {
         loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); 
-        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE;  
+        uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);  
-    } else {
-        loopDurationD = 0;
-        uncertaintyConstantD = 3;
     }
 
     /* anything with worse ratio in feas is definitely worse, discard */
@@ -1841,8 +1825,8 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* optimistic assumption of benchres.result */
     resultMax = benchres.result;
-    resultMax.cSpeed *= uncertaintyConstantC;
+    resultMax.cSpeed *= uncertaintyConstantC * VARIANCE;
-    resultMax.dSpeed *= uncertaintyConstantD;
+    resultMax.dSpeed *= uncertaintyConstantD * VARIANCE;
 
     /* disregard infeasible results in feas mode */
     /* disregard if resultMax < winner in infeas mode */
@@ -2125,7 +2109,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
-static int g_maxTries = 5;
+static int g_maxTries = 3;
 #define TRY_DECAY 1
 
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
@@ -2140,8 +2124,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     contexts_t ctx;
     buffers_t buf;
 
-    g_time = UTIL_getTime();
-
     /* Init */
     if(!cParamValid(paramTarget)) {
         return 1;
@@ -2234,6 +2216,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
 
+    /* Don't want it to return anything worse than the best known result */
     if(g_singleRun) {
         BMK_result_t res;
         g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
@@ -2450,6 +2433,8 @@ int main(int argc, const char** argv)
 
     assert(argc>=1);   /* for exename */
 
+    g_time = UTIL_getTime();
+
     /* Welcome message */
     DISPLAY(WELCOME_MESSAGE);