lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-07 06:23:00 +03:00
Code Activity

Merge pull request #1298 from facebook/bench

Browse Source 
Refactored bench.c
This commit is contained in:
Yann Collet
2018-08-28 12:25:02 -07:00
committed by GitHub
parent 883fbf60bc 55affc09de
commit b37a0a6bde
7 changed files with 1297 additions and 1079 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/compress/zstd_compress.c
View File

@@ -1334,8 +1334,7 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
} }
/* copy dictionary offsets */ /* copy dictionary offsets */
{ { ZSTD_matchState_t const* srcMatchState = &cdict->matchState;
ZSTD_matchState_t const* srcMatchState = &cdict->matchState;
ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState;
dstMatchState->window = srcMatchState->window; dstMatchState->window = srcMatchState->window;
dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;

930
programs/bench.c
View File

File diff suppressed because it is too large Load Diff

330
programs/bench.h
View File

@@ -15,59 +15,82 @@ extern "C" {
#ifndef BENCH_H_121279284357 #ifndef BENCH_H_121279284357
#define BENCH_H_121279284357 #define BENCH_H_121279284357
/* === Dependencies === */
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "zstd.h" /* ZSTD_compressionParameters */ #include "zstd.h" /* ZSTD_compressionParameters */
/* Creates a struct of type typeName with an int type .error field
* and a .result field of some baseType. Functions with return /* === Constants === */
* typeName pass a successful result with .error = 0 and .result
* with the intended result, while returning an error will result #define MB_UNIT 1000000
* in .error != 0.
/* === Benchmark functions === */
/* Creates a variant `typeName`, able to express "error or valid result".
* Functions with return type `typeName`
* must first check if result is valid, using BMK_isSuccessful_*(),
* and only then can extract `baseType`.
*/ */
#define ERROR_STRUCT(baseType, typeName) typedef struct { \ #define VARIANT_ERROR_RESULT(baseType, variantName) \
baseType result; \ \
int error; \ typedef struct { \
} typeName baseType internal_never_use_directly; \
int tag; \
} variantName
typedef struct { typedef struct {
size_t cSize; size_t cSize;
U64 cSpeed; /* bytes / sec */ unsigned long long cSpeed; /* bytes / sec */
U64 dSpeed; unsigned long long dSpeed;
size_t cMem; size_t cMem; /* ? what is reported ? */
} BMK_result_t; } BMK_benchResult_t;
ERROR_STRUCT(BMK_result_t, BMK_return_t); VARIANT_ERROR_RESULT(BMK_benchResult_t, BMK_benchOutcome_t);
/* called in cli */ /* check first if the return structure represents an error or a valid result */
/* Loads files in fileNamesTable into memory, as well as a dictionary int BMK_isSuccessful_benchOutcome(BMK_benchOutcome_t outcome);
* from dictFileName, and then uses benchMem */
/* fileNamesTable - name of files to benchmark /* extract result from variant type.
* nbFiles - number of files (size of fileNamesTable), must be > 0 * note : this function will abort() program execution if result is not valid
* dictFileName - name of dictionary file to load * check result validity first, by using BMK_isSuccessful_benchOutcome()
* cLevel - compression level to benchmark, errors if invalid */
* compressionParams - basic compression Parameters BMK_benchResult_t BMK_extract_benchResult(BMK_benchOutcome_t outcome);
* displayLevel - what gets printed
/*! BMK_benchFiles() -- called by zstdcli */
/* Loads files from fileNamesTable into memory,
* and an optional dictionary from dictFileName (can be NULL),
* then uses benchMem().
* fileNamesTable - name of files to benchmark.
* nbFiles - number of files (size of fileNamesTable), must be > 0.
* dictFileName - name of dictionary file to load.
* cLevel - compression level to benchmark, errors if invalid.
* compressionParams - advanced compression Parameters.
* displayLevel - what gets printed:
* 0 : no display; * 0 : no display;
* 1 : errors; * 1 : errors;
* 2 : + result + interaction + warnings; * 2 : + result + interaction + warnings;
* 3 : + progression; * 3 : + information;
* 4 : + information * 4 : + debug
* return * @return:
* .error will give a nonzero error value if an error has occured * a variant, which expresses either an error, or a valid result.
* .result - if .error = 0, .result will return the time taken to compression speed * Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original * If yes, extract the valid result with BMK_extract_benchResult(),
* file * it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
*/ */
BMK_return_t BMK_benchFiles(const char* const * const fileNamesTable, unsigned const nbFiles, BMK_benchOutcome_t BMK_benchFiles(
const char* const dictFileName, const char* const * fileNamesTable, unsigned nbFiles,
int const cLevel, const ZSTD_compressionParameters* const compressionParams, const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel); int displayLevel);
typedef enum {
BMK_timeMode = 0,
BMK_iterMode = 1
} BMK_loopMode_t;
typedef enum { typedef enum {
BMK_both = 0, BMK_both = 0,
@@ -77,14 +100,13 @@ typedef enum {
typedef struct { typedef struct {
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */ BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */
BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */
unsigned nbSeconds; /* default timing is in nbSeconds */ unsigned nbSeconds; /* default timing is in nbSeconds */
size_t blockSize; /* Maximum allowable size of a block*/ size_t blockSize; /* Maximum size of each block*/
unsigned nbWorkers; /* multithreading */ unsigned nbWorkers; /* multithreading */
unsigned realTime; /* real time priority */ unsigned realTime; /* real time priority */
int additionalParam; /* used by python speed benchmark */ int additionalParam; /* used by python speed benchmark */
unsigned ldmFlag; /* enables long distance matching */ unsigned ldmFlag; /* enables long distance matching */
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md for meaning */ unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md */
unsigned ldmHashLog; unsigned ldmHashLog;
unsigned ldmBucketSizeLog; unsigned ldmBucketSizeLog;
unsigned ldmHashEveryLog; unsigned ldmHashEveryLog;
@@ -93,132 +115,186 @@ typedef struct {
/* returns default parameters used by nonAdvanced functions */ /* returns default parameters used by nonAdvanced functions */
BMK_advancedParams_t BMK_initAdvancedParams(void); BMK_advancedParams_t BMK_initAdvancedParams(void);
/* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */ /*! BMK_benchFilesAdvanced():
BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, unsigned const nbFiles, * Same as BMK_benchFiles(),
const char* const dictFileName, * with more controls, provided through advancedParams_t structure */
int const cLevel, const ZSTD_compressionParameters* const compressionParams, BMK_benchOutcome_t BMK_benchFilesAdvanced(
int displayLevel, const BMK_advancedParams_t* const adv); const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
/* called in cli */ /*! BMK_syntheticTest() -- called from zstdcli */
/* Generates a sample with datagen with the compressibility argument*/ /* Generates a sample with datagen, using compressibility argument */
/* cLevel - compression level to benchmark, errors if invalid /* cLevel - compression level to benchmark, errors if invalid
* compressibility - determines compressibility of sample * compressibility - determines compressibility of sample
* compressionParams - basic compression Parameters * compressionParams - basic compression Parameters
* displayLevel - see benchFiles * displayLevel - see benchFiles
* adv - see advanced_Params_t * adv - see advanced_Params_t
* return * @return:
* .error will give a nonzero error value if an error has occured * a variant, which expresses either an error, or a valid result.
* .result - if .error = 0, .result will return the time taken to compression speed * Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* (.cSpeed), decompression speed (.dSpeed), and compressed size (.cSize) of the original * If yes, extract the valid result with BMK_extract_benchResult(),
* file * it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
*/ */
BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, BMK_benchOutcome_t BMK_syntheticTest(
int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t * const adv); int displayLevel, const BMK_advancedParams_t* adv);
/* basic benchmarking function, called in paramgrill
* applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
* with specific compression parameters specified by other arguments using benchFunction /* === Benchmark Zstandard in a memory-to-memory scenario === */
* (cLevel, comprParams + adv in advanced Mode) */
/* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode /** BMK_benchMem() -- core benchmarking function, called in paramgrill
* srcSize - size of data in srcBuffer * applies ZSTD_compress_generic() and ZSTD_decompress_generic() on data in srcBuffer
* cLevel - compression level * with specific compression parameters provided by other arguments using benchFunction
* comprParams - basic compression parameters * (cLevel, comprParams + adv in advanced Mode) */
* dictBuffer - a dictionary if used, null otherwise /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
* dictBufferSize - size of dictBuffer, 0 otherwise * srcSize - size of data in srcBuffer
* diplayLevel - see BMK_benchFiles * fileSizes - srcBuffer is considered cut into 1+ segments, to compress separately.
* displayName - name used by display * note : sum(fileSizes) must be == srcSize. (<== ensure it's properly checked)
* return * nbFiles - nb of segments
* .error will give a nonzero value if an error has occured * cLevel - compression level
* .result - if .error = 0, will give the same results as benchFiles * comprParams - basic compression parameters
* but for the data stored in srcBuffer * dictBuffer - a dictionary if used, null otherwise
* dictBufferSize - size of dictBuffer, 0 otherwise
* diplayLevel - see BMK_benchFiles
* displayName - name used by display
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
*/ */
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams, int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize, const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName); int displayLevel, const char* displayName);
/* See benchMem for normal parameter uses and return, see advancedParams_t for adv /* BMK_benchMemAdvanced() : same as BMK_benchMem()
* with following additional options :
* dstBuffer - destination buffer to write compressed output in, NULL if none provided. * dstBuffer - destination buffer to write compressed output in, NULL if none provided.
* dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
* adv = see advancedParams_t
*/ */
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstCapacity, void* dstBuffer, size_t dstCapacity,
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams, int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize, const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName, int displayLevel, const char* displayName,
const BMK_advancedParams_t* adv); const BMK_advancedParams_t* adv);
/* ==== Benchmarking any function, iterated on a set of blocks ==== */
typedef struct { typedef struct {
size_t sumOfReturn; /* sum of return values */ unsigned long long nanoSecPerRun; /* time per iteration */
U64 nanoSecPerRun; /* time per iteration */ size_t sumOfReturn; /* sum of return values */
} BMK_customResult_t; } BMK_runTime_t;
ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t); VARIANT_ERROR_RESULT(BMK_runTime_t, BMK_runOutcome_t);
typedef size_t (*BMK_benchFn_t)(const void*, size_t, void*, size_t, void*); /* check first if the return structure represents an error or a valid result */
typedef size_t (*BMK_initFn_t)(void*); int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome);
/* This function times the execution of 2 argument functions, benchFn and initFn */ /* extract result from variant type.
* note : this function will abort() program execution if result is not valid
* check result validity first, by using BMK_isSuccessful_runOutcome()
*/
BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome);
typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload);
typedef size_t (*BMK_initFn_t)(void* initPayload);
/* BMK_benchFunction() :
* This function times the execution of 2 argument functions, benchFn and initFn */
/* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) /* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
* is run nbLoops times * is run nbLoops times
* initFn - (*initFn)(initPayload) is run once per benchmark at the beginning. This argument can * initFn - (*initFn)(initPayload) is run once per benchmark, at the beginning.
* be NULL, in which case nothing is run. * This argument can be NULL, in which case nothing is run.
* blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities) * blockCount - number of blocks. Size of all array parameters : srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults
* srcBuffers - an array of buffers to be operated on by benchFn * srcBuffers - an array of buffers to be operated on by benchFn
* srcSizes - an array of the sizes of above buffers * srcSizes - an array of the sizes of above buffers
* dstBuffers - an array of buffers to be written into by benchFn * dstBuffers - an array of buffers to be written into by benchFn
* dstCapacities - an array of the capacities of above buffers * dstCapacities - an array of the capacities of above buffers
* blockResults - the return value of benchFn called on each block. * blockResults - store the return value of benchFn for each block. Optional. Use NULL if this result is not requested.
* nbLoops - defines number of times benchFn is run. * nbLoops - defines number of times benchFn is run.
* assumed array of size blockCount, will have compressed size of each block written to it. * @return: a variant, which express either an error, or can generate a valid BMK_runTime_t result.
* return * Use BMK_isSuccessful_runOutcome() to check if function was successful.
* .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return * If yes, extract the result with BMK_extract_runTime(),
* of the calls to initFn and benchFn, or if benchFunction errors internally * it will contain :
* .result - if .error = 0, then .result will contain the sum of all return values of * .sumOfReturn : the sum of all return values of benchFn through all of blocks
* benchFn on the first iteration through all of the blocks (.sumOfReturn) and also * .nanoSecPerRun : time per run of benchFn + (time for initFn / nbLoops)
* the time per run of benchFn (.nanoSecPerRun). For the former, this * .sumOfReturn is generally intended for functions which return a # of bytes written into dstBuffer,
* is generally intended to be used on functions which return the # of bytes written * in which case, this value will be the total amount of bytes written into dstBuffer.
* into dstBuffer, hence this value will be the total amount of bytes written to
* dstBuffer.
*/ */
BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload, BMK_runOutcome_t BMK_benchFunction(
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload, BMK_initFn_t initFn, void* initPayload,
size_t blockCount, size_t blockCount,
const void* const * const srcBuffers, const size_t* srcSizes, const void *const * srcBuffers, const size_t* srcSizes,
void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults, void *const * dstBuffers, const size_t* dstCapacities,
size_t* blockResults,
unsigned nbLoops); unsigned nbLoops);
/* state information needed to advance computation for benchFunctionTimed */
typedef struct BMK_timeState_t BMK_timedFnState_t;
/* initializes timeState object with desired number of seconds */
BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds);
/* resets existing timeState object */
void BMK_resetTimeState(BMK_timedFnState_t*, unsigned nbSeconds);
/* deletes timeState object */
void BMK_freeTimeState(BMK_timedFnState_t* state);
typedef struct { /* ==== Benchmark any function, providing intermediate results ==== */
BMK_customReturn_t result;
int completed;
} BMK_customTimedReturn_t;
/* /* state information tracking benchmark session */
* Benchmarks custom functions like BMK_benchFunction(), but runs for nbSeconds seconds rather than a fixed number of loops typedef struct BMK_timedFnState_s BMK_timedFnState_t;
* arguments mostly the same other than BMK_benchFunction()
* Usage - benchFunctionTimed will return in approximately one second. Keep calling BMK_benchFunctionTimed() until the return's completed field = 1. /* BMK_createTimedFnState() and BMK_resetTimedFnState() :
* to continue updating intermediate result. Intermediate return values are returned by the function. * Create/Set BMK_timedFnState_t for next benchmark session,
* which shall last a minimum of total_ms milliseconds,
* producing intermediate results, paced at interval of (approximately) run_ms.
*/ */
BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms);
BMK_benchFn_t benchFn, void* benchPayload, void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms);
BMK_initFn_t initFn, void* initPayload, void BMK_freeTimedFnState(BMK_timedFnState_t* state);
size_t blockCount,
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults); /* Tells if duration of all benchmark runs has exceeded total_ms
*/
int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState);
/* BMK_benchTimedFn() :
* Similar to BMK_benchFunction(), most arguments being identical.
* Automatically determines `nbLoops` so that each result is regularly produced at interval of about run_ms.
* Note : minimum `nbLoops` is 1, therefore a run may last more than run_ms, and possibly even more than total_ms.
* Usage - initialize timedFnState, select benchmark duration (total_ms) and each measurement duration (run_ms)
* call BMK_benchTimedFn() repetitively, each measurement is supposed to last about run_ms
* Check if total time budget is spent or exceeded, using BMK_isCompleted_TimedFn()
*/
BMK_runOutcome_t BMK_benchTimedFn(
BMK_timedFnState_t* timedFnState,
BMK_benchFn_t benchFn, void* benchPayload,
BMK_initFn_t initFn, void* initPayload,
size_t blockCount,
const void *const * srcBlockBuffers, const size_t* srcBlockSizes,
void *const * dstBlockBuffers, const size_t* dstBlockCapacities,
size_t* blockResults);
#endif /* BENCH_H_121279284357 */ #endif /* BENCH_H_121279284357 */

4
programs/zstdcli.c
View File

@@ -901,13 +901,13 @@ int main(int argCount, const char* argv[])
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel) if (cLevelLast > cLevel)
DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if(filenameIdx) { if(filenameIdx) {
if(separateFiles) { if(separateFiles) {
unsigned i; unsigned i;
for(i = 0; i < filenameIdx; i++) { for(i = 0; i < filenameIdx; i++) {
int c; int c;
DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]); DISPLAYLEVEL(3, "Benchmarking %s \n", filenameTable[i]);
for(c = cLevel; c <= cLevelLast; c++) { for(c = cLevel; c <= cLevelLast; c++) {
BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams); BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
} }

2
tests/Makefile
View File

@@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll
zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
paramgrill : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements
paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT) $(CC) $(FLAGS) $^ -lm -o $@$(EXT)

413
tests/fullbench.c
View File

@@ -51,6 +51,8 @@
#define COMPRESSIBILITY_DEFAULT 0.50 #define COMPRESSIBILITY_DEFAULT 0.50
static const size_t g_sampleSize = 10000000; static const size_t g_sampleSize = 10000000;
#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
/*_************************************ /*_************************************
* Macros * Macros
@@ -92,63 +94,30 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t) requiredMem; return (size_t) requiredMem;
} }
/*_*******************************************************
* Argument Parsing
*********************************************************/
#define ERROR_OUT(msg) { DISPLAY("%s \n", msg); exit(1); }
static unsigned readU32FromChar(const char** stringPtr)
{
const char errorMsg[] = "error: numeric value too large";
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
unsigned const max = (((unsigned)(-1)) / 10) - 1;
if (result > max) ERROR_OUT(errorMsg);
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
if (result > maxK) ERROR_OUT(errorMsg);
result <<= 10;
if (**stringPtr=='M') {
if (result > maxK) ERROR_OUT(errorMsg);
result <<= 10;
}
(*stringPtr)++; /* skip `K` or `M` */
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
return result;
}
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
if (result) *stringPtr += comSize;
return result;
}
/*_******************************************************* /*_*******************************************************
* Benchmark wrappers * Benchmark wrappers
*********************************************************/ *********************************************************/
static ZSTD_CCtx* g_zcc = NULL; static ZSTD_CCtx* g_zcc = NULL;
size_t local_ZSTD_compress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) static size_t
local_ZSTD_compress(const void* src, size_t srcSize,
void* dst, size_t dstSize,
void* buff2)
{ {
ZSTD_parameters p; ZSTD_parameters p;
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0}; ZSTD_frameParameters f = { 1 /* contentSizeHeader*/, 0, 0 };
p.fParams = f; p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2; p.cParams = *(ZSTD_compressionParameters*)buff2;
return ZSTD_compress_advanced (g_zcc,dst, dstSize, src, srcSize, NULL ,0, p); return ZSTD_compress_advanced (g_zcc, dst, dstSize, src, srcSize, NULL ,0, p);
//return ZSTD_compress(dst, dstSize, src, srcSize, cLevel); //return ZSTD_compress(dst, dstSize, src, srcSize, cLevel);
} }
static size_t g_cSize = 0; static size_t g_cSize = 0;
size_t local_ZSTD_decompress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) static size_t local_ZSTD_decompress(const void* src, size_t srcSize,
void* dst, size_t dstSize,
void* buff2)
{ {
(void)src; (void)srcSize; (void)src; (void)srcSize;
return ZSTD_decompress(dst, dstSize, buff2, g_cSize); return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
@@ -174,7 +143,10 @@ size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, s
#endif #endif
static ZSTD_CStream* g_cstream= NULL; static ZSTD_CStream* g_cstream= NULL;
size_t local_ZSTD_compressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) static size_t
local_ZSTD_compressStream(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{ {
ZSTD_outBuffer buffOut; ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn; ZSTD_inBuffer buffIn;
@@ -194,7 +166,10 @@ size_t local_ZSTD_compressStream(const void* src, size_t srcSize, void* dst, siz
return buffOut.pos; return buffOut.pos;
} }
static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) static size_t
local_ZSTD_compress_generic_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{ {
ZSTD_outBuffer buffOut; ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn; ZSTD_inBuffer buffIn;
@@ -209,7 +184,10 @@ static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, v
return buffOut.pos; return buffOut.pos;
} }
static size_t local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) static size_t
local_ZSTD_compress_generic_continue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{ {
ZSTD_outBuffer buffOut; ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn; ZSTD_inBuffer buffIn;
@@ -225,7 +203,10 @@ static size_t local_ZSTD_compress_generic_continue(const void* src, size_t srcSi
return buffOut.pos; return buffOut.pos;
} }
static size_t local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) static size_t
local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{ {
ZSTD_outBuffer buffOut; ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn; ZSTD_inBuffer buffIn;
@@ -241,7 +222,10 @@ static size_t local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize
return buffOut.pos; return buffOut.pos;
} }
static size_t local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) static size_t
local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{ {
ZSTD_outBuffer buffOut; ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn; ZSTD_inBuffer buffIn;
@@ -259,7 +243,10 @@ static size_t local_ZSTD_compress_generic_T2_continue(const void* src, size_t sr
} }
static ZSTD_DStream* g_dstream= NULL; static ZSTD_DStream* g_dstream= NULL;
static size_t local_ZSTD_decompressStream(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) static size_t
local_ZSTD_decompressStream(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{ {
ZSTD_outBuffer buffOut; ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn; ZSTD_inBuffer buffIn;
@@ -276,10 +263,12 @@ static size_t local_ZSTD_decompressStream(const void* src, size_t srcSize, void*
} }
#ifndef ZSTD_DLL_IMPORT #ifndef ZSTD_DLL_IMPORT
size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) size_t local_ZSTD_compressContinue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{ {
ZSTD_parameters p; ZSTD_parameters p;
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0}; ZSTD_frameParameters f = { 1 /* contentSizeHeader*/, 0, 0 };
p.fParams = f; p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2; p.cParams = *(ZSTD_compressionParameters*)buff2;
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize); ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
@@ -287,26 +276,38 @@ size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, s
} }
#define FIRST_BLOCK_SIZE 8 #define FIRST_BLOCK_SIZE 8
size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{ {
BYTE firstBlockBuf[FIRST_BLOCK_SIZE]; BYTE firstBlockBuf[FIRST_BLOCK_SIZE];
ZSTD_parameters p; ZSTD_parameters p;
ZSTD_frameParameters f = {1 , 0, 0}; ZSTD_frameParameters f = { 1, 0, 0 };
p.fParams = f; p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2; p.cParams = *(ZSTD_compressionParameters*)buff2;
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize); ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE); memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE);
{ size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE); { size_t const compressResult = ZSTD_compressContinue(g_zcc,
if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; } dst, dstCapacity,
firstBlockBuf, FIRST_BLOCK_SIZE);
if (ZSTD_isError(compressResult)) {
DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n",
ZSTD_getErrorName(compressResult));
return compressResult;
}
dst = (BYTE*)dst + compressResult; dst = (BYTE*)dst + compressResult;
dstCapacity -= compressResult; dstCapacity -= compressResult;
} }
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, (const BYTE*)src + FIRST_BLOCK_SIZE, srcSize - FIRST_BLOCK_SIZE); return ZSTD_compressEnd(g_zcc, dst, dstCapacity,
(const BYTE*)src + FIRST_BLOCK_SIZE,
srcSize - FIRST_BLOCK_SIZE);
} }
size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2) size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{ {
size_t regeneratedSize = 0; size_t regeneratedSize = 0;
const BYTE* ip = (const BYTE*)buff2; const BYTE* ip = (const BYTE*)buff2;
@@ -314,7 +315,7 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst,
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
size_t remainingCapacity = dstCapacity; size_t remainingCapacity = dstCapacity;
(void)src; (void)srcSize; (void)src; (void)srcSize; /* unused */
ZSTD_decompressBegin(g_zdc); ZSTD_decompressBegin(g_zdc);
while (ip < iend) { while (ip < iend) {
size_t const iSize = ZSTD_nextSrcSizeToDecompress(g_zdc); size_t const iSize = ZSTD_nextSrcSizeToDecompress(g_zdc);
@@ -333,14 +334,16 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst,
/*_******************************************************* /*_*******************************************************
* Bench functions * Bench functions
*********************************************************/ *********************************************************/
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) static size_t benchMem(U32 benchNb,
const void* src, size_t srcSize,
int cLevel, ZSTD_compressionParameters cparams)
{ {
BYTE* dstBuff;
size_t dstBuffSize = ZSTD_compressBound(srcSize); size_t dstBuffSize = ZSTD_compressBound(srcSize);
void* buff2, *buff1; BYTE* dstBuff;
void* dstBuff2;
void* buff2;
const char* benchName; const char* benchName;
BMK_benchFn_t benchFunction; BMK_benchFn_t benchFunction;
BMK_customReturn_t r;
int errorcode = 0; int errorcode = 0;
/* Selection */ /* Selection */
@@ -393,56 +396,56 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel,
/* Allocation */ /* Allocation */
dstBuff = (BYTE*)malloc(dstBuffSize); dstBuff = (BYTE*)malloc(dstBuffSize);
buff2 = malloc(dstBuffSize); dstBuff2 = malloc(dstBuffSize);
if ((!dstBuff) || (!buff2)) { if ((!dstBuff) || (!dstBuff2)) {
DISPLAY("\nError: not enough memory!\n"); DISPLAY("\nError: not enough memory!\n");
free(dstBuff); free(buff2); free(dstBuff); free(dstBuff2);
return 12; return 12;
} }
buff1 = buff2; buff2 = dstBuff2;
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx(); if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx(); if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
if (g_cstream==NULL) g_cstream = ZSTD_createCStream(); if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
if (g_dstream==NULL) g_dstream = ZSTD_createDStream(); if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n" /* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n",
, cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog, cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog,
cparams->searchLength, cparams->targetLength, cparams->strategy);*/ cparams->searchLength, cparams->targetLength, cparams->strategy); */
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams->windowLog); ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams.windowLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams->hashLog); ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams.hashLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams->chainLog); ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams.chainLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams->searchLog); ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams->searchLength); ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams.searchLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams->targetLength); ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams->strategy); ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams->windowLog); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams.windowLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams->hashLog); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams.hashLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams->chainLog); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams.chainLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams->searchLog); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams->searchLength); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams.searchLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams->targetLength); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams->strategy); ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams.strategy);
/* Preparation */ /* Preparation */
switch(benchNb) switch(benchNb)
{ {
case 1: case 1:
buff2 = (void*)cparams; buff2 = &cparams;
break; break;
case 2: case 2:
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
break; break;
#ifndef ZSTD_DLL_IMPORT #ifndef ZSTD_DLL_IMPORT
case 11: case 11:
buff2 = (void*)cparams; buff2 = &cparams;
break; break;
case 12: case 12:
buff2 = (void*)cparams; buff2 = &cparams;
break; break;
case 13 : case 13 :
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
@@ -495,7 +498,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel,
goto _cleanOut; goto _cleanOut;
#endif #endif
case 41 : case 41 :
buff2 = (void*)cparams; buff2 = &cparams;
break; break;
case 42 : case 42 :
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
@@ -507,29 +510,50 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel,
default : ; default : ;
} }
/* warming up dstBuff */
/* warming up memory */
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } { size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; }
/* benchmark loop */ /* benchmark loop */
{ { BMK_timedFnState_t* const tfs = BMK_createTimedFnState(g_nbIterations * 1000, 1000);
void* dstBuffv = (void*)dstBuff; BMK_runTime_t bestResult;
r = BMK_benchFunction(benchFunction, buff2, bestResult.sumOfReturn = 0;
NULL, NULL, 1, &src, &srcSize, bestResult.nanoSecPerRun = (unsigned long long)(-1LL);
&dstBuffv, &dstBuffSize, NULL, g_nbIterations); assert(tfs != NULL);
if(r.error) { for (;;) {
DISPLAY("ERROR %d ! ! \n", r.error); void* const dstBuffv = dstBuff;
errorcode = r.error; BMK_runOutcome_t const bOutcome =
goto _cleanOut; BMK_benchTimedFn( tfs,
} benchFunction, buff2,
NULL, NULL, /* initFn */
1, /* blockCount */
&src, &srcSize,
&dstBuffv, &dstBuffSize,
NULL);
DISPLAY("%2u#Speed: %f MB/s - Size: %f MB - %s\n", benchNb, (double)srcSize / r.result.nanoSecPerRun * 1000, (double)r.result.sumOfReturn / 1000000, benchName); if (!BMK_isSuccessful_runOutcome(bOutcome)) {
DISPLAY("ERROR benchmarking function ! ! \n");
errorcode = 1;
goto _cleanOut;
}
{ BMK_runTime_t const newResult = BMK_extract_runTime(bOutcome);
if (newResult.nanoSecPerRun < bestResult.nanoSecPerRun )
bestResult.nanoSecPerRun = newResult.nanoSecPerRun;
DISPLAY("\r%2u#%-29.29s:%8.1f MB/s (%8u) ",
benchNb, benchName,
(double)srcSize * TIMELOOP_NANOSEC / bestResult.nanoSecPerRun / MB_UNIT,
(unsigned)newResult.sumOfReturn );
}
if ( BMK_isCompleted_TimedFn(tfs) ) break;
}
BMK_freeTimedFnState(tfs);
} }
DISPLAY("\n");
_cleanOut: _cleanOut:
free(buff1);
free(dstBuff); free(dstBuff);
free(dstBuff2);
ZSTD_freeCCtx(g_zcc); g_zcc=NULL; ZSTD_freeCCtx(g_zcc); g_zcc=NULL;
ZSTD_freeDCtx(g_zdc); g_zdc=NULL; ZSTD_freeDCtx(g_zdc); g_zdc=NULL;
ZSTD_freeCStream(g_cstream); g_cstream=NULL; ZSTD_freeCStream(g_cstream); g_cstream=NULL;
@@ -538,87 +562,138 @@ _cleanOut:
} }
static int benchSample(U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) static int benchSample(U32 benchNb,
int cLevel, ZSTD_compressionParameters cparams)
{ {
size_t const benchedSize = g_sampleSize; size_t const benchedSize = g_sampleSize;
const char* name = "Sample 10MiB"; const char* const name = "Sample 10MiB";
/* Allocation */ /* Allocation */
void* origBuff = malloc(benchedSize); void* const origBuff = malloc(benchedSize);
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; } if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; }
/* Fill buffer */ /* Fill buffer */
RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
/* bench */ /* bench */
DISPLAY("\r%79s\r", ""); DISPLAY("\r%70s\r", "");
DISPLAY(" %s : \n", name); DISPLAY(" %s : \n", name);
if (benchNb) if (benchNb) {
benchMem(origBuff, benchedSize, benchNb, cLevel, cparams); benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
else } else { /* 0 == run all tests */
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams); for (benchNb=0; benchNb<100; benchNb++) {
benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
} }
free(origBuff); free(origBuff);
return 0; return 0;
} }
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) static int benchFiles(U32 benchNb,
const char** fileNamesTable, const int nbFiles,
int cLevel, ZSTD_compressionParameters cparams)
{ {
/* Loop for each file */ /* Loop for each file */
int fileIdx; int fileIdx;
for (fileIdx=0; fileIdx<nbFiles; fileIdx++) { for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
const char* const inFileName = fileNamesTable[fileIdx]; const char* const inFileName = fileNamesTable[fileIdx];
FILE* const inFile = fopen( inFileName, "rb" ); FILE* const inFile = fopen( inFileName, "rb" );
U64 inFileSize;
size_t benchedSize; size_t benchedSize;
void* origBuff;
/* Check file existence */ /* Check file existence */
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
/* Memory allocation & restrictions */ /* Memory allocation & restrictions */
inFileSize = UTIL_getFileSize(inFileName); { U64 const inFileSize = UTIL_getFileSize(inFileName);
if (inFileSize == UTIL_FILESIZE_UNKNOWN) { if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAY( "Cannot measure size of %s\n", inFileName); DISPLAY( "Cannot measure size of %s\n", inFileName);
fclose(inFile); fclose(inFile);
return 11; return 11;
} }
benchedSize = BMK_findMaxMem(inFileSize*3) / 3; benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if ((U64)benchedSize > inFileSize)
if (benchedSize < inFileSize) benchedSize = (size_t)inFileSize;
DISPLAY("Not enough memory for '%s' full size; testing %u MB only...\n", inFileName, (U32)(benchedSize>>20)); if ((U64)benchedSize < inFileSize) {
DISPLAY("Not enough memory for '%s' full size; testing %u MB only... \n",
/* Alloc */ inFileName, (U32)(benchedSize>>20));
origBuff = malloc(benchedSize);
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; }
/* Fill input buffer */
DISPLAY("Loading %s... \r", inFileName);
{
size_t readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile);
if (readSize != benchedSize) {
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(origBuff);
return 13;
} } } }
/* bench */ /* Alloc */
DISPLAY("\r%79s\r", ""); { void* const origBuff = malloc(benchedSize);
DISPLAY(" %s : \n", inFileName); if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; }
if (benchNb)
benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
else
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
free(origBuff); /* Fill input buffer */
} DISPLAY("Loading %s... \r", inFileName);
{ size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile);
if (readSize != benchedSize) {
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(origBuff);
return 13;
} }
/* bench */
DISPLAY("\r%70s\r", ""); /* blank line */
DISPLAY(" %s : \n", inFileName);
if (benchNb) {
benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
} else {
for (benchNb=0; benchNb<100; benchNb++) {
benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
} }
free(origBuff);
} }
return 0; return 0;
} }
/*_*******************************************************
* Argument Parsing
*********************************************************/
#define ERROR_OUT(msg) { DISPLAY("%s \n", msg); exit(1); }
static unsigned readU32FromChar(const char** stringPtr)
{
const char errorMsg[] = "error: numeric value too large";
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
unsigned const max = (((unsigned)(-1)) / 10) - 1;
if (result > max) ERROR_OUT(errorMsg);
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
if (result > maxK) ERROR_OUT(errorMsg);
result <<= 10;
if (**stringPtr=='M') {
if (result > maxK) ERROR_OUT(errorMsg);
result <<= 10;
}
(*stringPtr)++; /* skip `K` or `M` */
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
return result;
}
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
if (result) *stringPtr += comSize;
return result;
}
/*_*******************************************************
* Command line
*********************************************************/
static int usage(const char* exename) static int usage(const char* exename)
{ {
DISPLAY( "Usage :\n"); DISPLAY( "Usage :\n");
@@ -649,8 +724,8 @@ static int badusage(const char* exename)
int main(int argc, const char** argv) int main(int argc, const char** argv)
{ {
int i, filenamesStart=0, result; int argNb, filenamesStart=0, result;
const char* exename = argv[0]; const char* const exename = argv[0];
const char* input_filename = NULL; const char* input_filename = NULL;
U32 benchNb = 0, main_pause = 0; U32 benchNb = 0, main_pause = 0;
int cLevel = DEFAULT_CLEVEL; int cLevel = DEFAULT_CLEVEL;
@@ -659,8 +734,8 @@ int main(int argc, const char** argv)
DISPLAY(WELCOME_MESSAGE); DISPLAY(WELCOME_MESSAGE);
if (argc<1) return badusage(exename); if (argc<1) return badusage(exename);
for(i=1; i<argc; i++) { for (argNb=1; argNb<argc; argNb++) {
const char* argument = argv[i]; const char* argument = argv[argNb];
assert(argument != NULL); assert(argument != NULL);
if (longCommandWArg(&argument, "--zstd=")) { if (longCommandWArg(&argument, "--zstd=")) {
@@ -677,12 +752,14 @@ int main(int argc, const char** argv)
return 1; return 1;
} }
/* check end of string */
if (argument[0] != 0) { if (argument[0] != 0) {
DISPLAY("invalid --zstd= format\n"); DISPLAY("invalid --zstd= format \n");
return 1; // check the end of string return 1;
} else { } else {
continue; continue;
} }
} else if (argument[0]=='-') { /* Commands (note : aggregated commands are allowed) */ } else if (argument[0]=='-') { /* Commands (note : aggregated commands are allowed) */
argument++; argument++;
while (argument[0]!=0) { while (argument[0]!=0) {
@@ -698,35 +775,27 @@ int main(int argc, const char** argv)
/* Select specific algorithm to bench */ /* Select specific algorithm to bench */
case 'b': case 'b':
{ argument++;
argument++; benchNb = readU32FromChar(&argument);
benchNb = readU32FromChar(&argument); break;
break;
}
/* Modify Nb Iterations */ /* Modify Nb Iterations */
case 'i': case 'i':
{ argument++;
argument++; BMK_SetNbIterations((int)readU32FromChar(&argument));
BMK_SetNbIterations((int)readU32FromChar(&argument));
}
break; break;
/* Select compressibility of synthetic sample */ /* Select compressibility of synthetic sample */
case 'P': case 'P':
{ argument++; argument++;
g_compressibility = (double)readU32FromChar(&argument) / 100.; g_compressibility = (double)readU32FromChar(&argument) / 100.;
}
break; break;
case 'l': case 'l':
{ argument++; argument++;
cLevel = readU32FromChar(&argument); cLevel = readU32FromChar(&argument);
cparams = ZSTD_getCParams(cLevel, 0, 0); cparams = ZSTD_getCParams(cLevel, 0, 0);
}
break; break;
/* Unknown command */ /* Unknown command */
default : return badusage(exename); default : return badusage(exename);
} }
@@ -735,15 +804,15 @@ int main(int argc, const char** argv)
} }
/* first provided filename is input */ /* first provided filename is input */
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } if (!input_filename) { input_filename=argument; filenamesStart=argNb; continue; }
} }
if (filenamesStart==0) /* no input file */ if (filenamesStart==0) /* no input file */
result = benchSample(benchNb, cLevel, &cparams); result = benchSample(benchNb, cLevel, cparams);
else else
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb, cLevel, &cparams); result = benchFiles(benchNb, argv+filenamesStart, argc-filenamesStart, cLevel, cparams);
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }

424
tests/paramgrill.c
View File

@@ -27,7 +27,8 @@
#include "util.h" #include "util.h"
#include "bench.h" #include "bench.h"
#include "zstd_errors.h" #include "zstd_errors.h"
#include "zstd_internal.h" #include "zstd_internal.h" /* should not be needed */
/*-************************************ /*-************************************
* Constants * Constants
@@ -46,6 +47,7 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t
static const U64 g_maxVariationTime = 60 * SEC_TO_MICRO; static const U64 g_maxVariationTime = 60 * SEC_TO_MICRO;
static const int g_maxNbVariations = 64; static const int g_maxNbVariations = 64;
/*-************************************ /*-************************************
* Macros * Macros
**************************************/ **************************************/
@@ -90,9 +92,9 @@ static const char* g_stratName[ZSTD_btultra+1] = {
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
/*-************************************ /*-************************************
* Setup for Adding new params * Setup for Adding new params
**************************************/ **************************************/
@@ -212,6 +214,7 @@ static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
} }
} }
/*-************************************ /*-************************************
* Benchmark Parameters/Global Variables * Benchmark Parameters/Global Variables
**************************************/ **************************************/
@@ -241,7 +244,7 @@ static U32 g_noSeed = 0;
static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
static double g_ratioMultiplier = 5.; static double g_ratioMultiplier = 5.;
static U32 g_strictness = PARAM_UNSET; /* range 1 - 100, measure of how strict */ static U32 g_strictness = PARAM_UNSET; /* range 1 - 100, measure of how strict */
static BMK_result_t g_lvltarget; static BMK_benchResult_t g_lvltarget;
typedef enum { typedef enum {
directMap, directMap,
@@ -258,7 +261,7 @@ typedef struct {
} memoTable_t; } memoTable_t;
typedef struct { typedef struct {
BMK_result_t result; BMK_benchResult_t result;
paramValues_t params; paramValues_t params;
} winnerInfo_t; } winnerInfo_t;
@@ -284,6 +287,7 @@ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSp
* g_clockGranularity * g_clockGranularity
*/ */
/*-******************************************************* /*-*******************************************************
* General Util Functions * General Util Functions
*********************************************************/ *********************************************************/
@@ -464,7 +468,7 @@ static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbC
static paramValues_t randomParams(void) static paramValues_t randomParams(void)
{ {
varInds_t v; paramValues_t p; varInds_t v; paramValues_t p;
for(v = 0; v <= NUM_PARAMS; v++) { for(v = 0; v < NUM_PARAMS; v++) {
p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
} }
return p; return p;
@@ -497,8 +501,11 @@ static void findClockGranularity(void) {
**************************************/ **************************************/
/* checks results are feasible */ /* checks results are feasible */
static int feasible(const BMK_result_t results, const constraint_t target) { static int feasible(const BMK_benchResult_t results, const constraint_t target) {
return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize); return (results.cSpeed >= target.cSpeed)
&& (results.dSpeed >= target.dSpeed)
&& (results.cMem <= target.cMem)
&& (!g_optmode || results.cSize <= g_lvltarget.cSize);
} }
/* hill climbing value for part 1 */ /* hill climbing value for part 1 */
@@ -507,7 +514,7 @@ static int feasible(const BMK_result_t results, const constraint_t target) {
* bonus to exceeding the constraint value. We also give linear ratio for compression ratio. * bonus to exceeding the constraint value. We also give linear ratio for compression ratio.
* The constant factors are experimental. * The constant factors are experimental.
*/ */
static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { static double resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_t target) {
double cs = 0., ds = 0., rt, cm = 0.; double cs = 0., ds = 0., rt, cm = 0.;
const double r1 = 1, r2 = 0.1, rtr = 0.5; const double r1 = 1, r2 = 0.1, rtr = 0.5;
double ret; double ret;
@@ -523,7 +530,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co
} }
/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ /* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) { static double resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes) {
double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1; double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1;
double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1; double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
@@ -533,7 +540,7 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe
} }
/* return true if r2 strictly better than r1 */ /* return true if r2 strictly better than r1 */
static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { static int compareResultLT(const BMK_benchResult_t result1, const BMK_benchResult_t result2, const constraint_t target, size_t srcSize) {
if(feasible(result1, target) && feasible(result2, target)) { if(feasible(result1, target) && feasible(result2, target)) {
if(g_optmode) { if(g_optmode) {
return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget); return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
@@ -625,37 +632,44 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
varInds_t v; varInds_t v;
int first = 1; int first = 1;
fprintf(f,"--zstd="); fprintf(f,"--zstd=");
for(v = 0; v < NUM_PARAMS; v++) { for (v = 0; v < NUM_PARAMS; v++) {
if(g_silenceParams[v]) { continue; } if (g_silenceParams[v]) { continue; }
if(!first) { fprintf(f, ","); } if (!first) { fprintf(f, ","); }
fprintf(f,"%s=", g_paramNames[v]); fprintf(f,"%s=", g_paramNames[v]);
if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); } if (v == strt_ind) { fprintf(f,"%u", params.vals[v]); }
else { displayParamVal(f, v, params.vals[v], 0); } else { displayParamVal(f, v, params.vals[v], 0); }
first = 0; first = 0;
} }
fprintf(f, "\n"); fprintf(f, "\n");
} }
static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize)
varInds_t v; {
int first = 1; varInds_t v;
res.params = cParamUnsetMin(res.params); int first = 1;
fprintf(f," {"); res.params = cParamUnsetMin(res.params);
for(v = 0; v < NUM_PARAMS; v++) { fprintf(f, " {");
if(g_silenceParams[v]) { continue; } for (v = 0; v < NUM_PARAMS; v++) {
if(!first) { fprintf(f, ","); } if (g_silenceParams[v]) { continue; }
displayParamVal(f, v, res.params.vals[v], 3); if (!first) { fprintf(f, ","); }
first = 0; displayParamVal(f, v, res.params.vals[v], 3);
} first = 0;
}
fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", { double const ratio = res.result.cSize ?
(double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); (double)srcSize / res.result.cSize : 0;
double const cSpeedMBps = (double)res.result.cSpeed / MB_UNIT;
double const dSpeedMBps = (double)res.result.dSpeed / MB_UNIT;
fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
ratio, cSpeedMBps, dSpeedMBps);
}
} }
/* Writes to f the results of a parameter benchmark */ /* Writes to f the results of a parameter benchmark */
/* when used with --optimize, will only print results better than previously discovered */ /* when used with --optimize, will only print results better than previously discovered */
static void BMK_printWinner(FILE* f, const int cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) static void BMK_printWinner(FILE* f, const int cLevel, const BMK_benchResult_t result, const paramValues_t params, const size_t srcSize)
{ {
char lvlstr[15] = "Custom Level"; char lvlstr[15] = "Custom Level";
winnerInfo_t w; winnerInfo_t w;
@@ -687,7 +701,7 @@ static void BMK_printWinner(FILE* f, const int cLevel, const BMK_result_t result
#define SPEED_RESULT 4 #define SPEED_RESULT 4
#define SIZE_RESULT 5 #define SIZE_RESULT 5
/* maybe have epsilon-eq to limit table size? */ /* maybe have epsilon-eq to limit table size? */
static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) { static int speedSizeCompare(const BMK_benchResult_t r1, const BMK_benchResult_t r2) {
if(r1.cSpeed < r2.cSpeed) { if(r1.cSpeed < r2.cSpeed) {
if(r1.cSize >= r2.cSize) { if(r1.cSize >= r2.cSize) {
return BETTER_RESULT; return BETTER_RESULT;
@@ -704,7 +718,7 @@ static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) {
/* 0 for insertion, 1 for no insert */ /* 0 for insertion, 1 for no insert */
/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) { static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) {
BMK_result_t r = w.result; BMK_benchResult_t r = w.result;
winner_ll_node* cur_node = g_winners; winner_ll_node* cur_node = g_winners;
/* first node to insert */ /* first node to insert */
if(!feasible(r, targetConstraints)) { if(!feasible(r, targetConstraints)) {
@@ -797,7 +811,7 @@ static int insertWinner(const winnerInfo_t w, const constraint_t targetConstrain
} }
} }
static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
{ {
/* global winner used for constraints */ /* global winner used for constraints */
/* cSize, cSpeed, dSpeed, cMem */ /* cSize, cSpeed, dSpeed, cMem */
@@ -833,7 +847,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
} }
fprintf(f, "================================\n"); fprintf(f, "================================\n");
fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
(double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB)); (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / MB_UNIT);
fprintf(f, "Overall Winner: \n"); fprintf(f, "Overall Winner: \n");
@@ -871,7 +885,7 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t
*********************************************************/ *********************************************************/
typedef struct { typedef struct {
ZSTD_CCtx* ctx; ZSTD_CCtx* cctx;
const void* dictBuffer; const void* dictBuffer;
size_t dictBufferSize; size_t dictBufferSize;
int cLevel; int cLevel;
@@ -881,15 +895,15 @@ typedef struct {
static size_t local_initCCtx(void* payload) { static size_t local_initCCtx(void* payload) {
const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
varInds_t i; varInds_t i;
ZSTD_CCtx_reset(ag->ctx); ZSTD_CCtx_reset(ag->cctx);
ZSTD_CCtx_resetParameters(ag->ctx); ZSTD_CCtx_resetParameters(ag->cctx);
ZSTD_CCtx_setParameter(ag->ctx, ZSTD_p_compressionLevel, ag->cLevel); ZSTD_CCtx_setParameter(ag->cctx, ZSTD_p_compressionLevel, ag->cLevel);
for(i = 0; i < NUM_PARAMS; i++) { for(i = 0; i < NUM_PARAMS; i++) {
if(ag->comprParams->vals[i] != PARAM_UNSET) if(ag->comprParams->vals[i] != PARAM_UNSET)
ZSTD_CCtx_setParameter(ag->ctx, cctxSetParamTable[i], ag->comprParams->vals[i]); ZSTD_CCtx_setParameter(ag->cctx, cctxSetParamTable[i], ag->comprParams->vals[i]);
} }
ZSTD_CCtx_loadDictionary(ag->ctx, ag->dictBuffer, ag->dictBufferSize); ZSTD_CCtx_loadDictionary(ag->cctx, ag->dictBuffer, ag->dictBufferSize);
return 0; return 0;
} }
@@ -1378,16 +1392,17 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa
/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */
/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */
/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
/* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ /* nbSeconds used in same way as in BMK_advancedParams_t */
/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */
/* dictionary nullable, nothing else though. */ /* dictionary nullable, nothing else though. */
static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, /* note : it would be better if this function was in bench.c, sharing code with benchMemAdvanced(), since it's technically a part of it */
const int cLevel, const paramValues_t* comprParams, static BMK_benchOutcome_t
const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { BMK_benchMemInvertible( buffers_t buf, contexts_t ctx,
int cLevel, const paramValues_t* comprParams,
BMK_mode_t mode, unsigned nbSeconds)
{
U32 i; U32 i;
BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; BMK_benchResult_t bResult;
const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs;
size_t const *const srcSizes = buf.srcSizes; size_t const *const srcSizes = buf.srcSizes;
void** const dstPtrs = buf.dstPtrs; void** const dstPtrs = buf.dstPtrs;
@@ -1402,9 +1417,12 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
ZSTD_CCtx* cctx = ctx.cctx; ZSTD_CCtx* cctx = ctx.cctx;
ZSTD_DCtx* dctx = ctx.dctx; ZSTD_DCtx* dctx = ctx.dctx;
/* init */
memset(&bResult, 0, sizeof(bResult));
/* warmimg up memory */ /* warmimg up memory */
for(i = 0; i < buf.nbBlocks; i++) { for (i = 0; i < buf.nbBlocks; i++) {
if(mode != BMK_decodeOnly) { if (mode != BMK_decodeOnly) {
RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1); RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1);
} else { } else {
RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1); RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1);
@@ -1414,9 +1432,13 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
/* Bench */ /* Bench */
{ {
/* init args */ /* init args */
int compressionCompleted = (mode == BMK_decodeOnly);
int decompressionCompleted = (mode == BMK_compressOnly);
BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(nbSeconds * 1000, 1000);
BMK_timedFnState_t* timeStateDecompress = BMK_createTimedFnState(nbSeconds * 1000, 1000);
BMK_initCCtxArgs cctxprep; BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep; BMK_initDCtxArgs dctxprep;
cctxprep.ctx = cctx; cctxprep.cctx = cctx;
cctxprep.dictBuffer = dictBuffer; cctxprep.dictBuffer = dictBuffer;
cctxprep.dictBufferSize = dictBufferSize; cctxprep.dictBufferSize = dictBufferSize;
cctxprep.cLevel = cLevel; cctxprep.cLevel = cLevel;
@@ -1425,130 +1447,115 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
dctxprep.dictBuffer = dictBuffer; dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize; dctxprep.dictBufferSize = dictBufferSize;
if(loopMode == BMK_timeMode) { assert(timeStateCompress != NULL);
BMK_customTimedReturn_t intermediateResultCompress; assert(timeStateDecompress != NULL);
BMK_customTimedReturn_t intermediateResultDecompress; while(!compressionCompleted) {
BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); BMK_runOutcome_t const cOutcome = BMK_benchTimedFn(timeStateCompress,
BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); &local_defaultCompress, cctx,
if(mode == BMK_compressOnly) { &local_initCCtx, &cctxprep,
intermediateResultCompress.completed = 0; nbBlocks,
intermediateResultDecompress.completed = 1; srcPtrs, srcSizes,
} else if (mode == BMK_decodeOnly) { dstPtrs, dstCapacities,
intermediateResultCompress.completed = 1; dstSizes);
intermediateResultDecompress.completed = 0;
} else { /* both */ if (!BMK_isSuccessful_runOutcome(cOutcome)) {
intermediateResultCompress.completed = 0; BMK_benchOutcome_t bOut;
intermediateResultDecompress.completed = 0; memset(&bOut, 0, sizeof(bOut));
bOut.tag = 1; /* should rather be a function or a constant */
BMK_freeTimedFnState(timeStateCompress);
BMK_freeTimedFnState(timeStateDecompress);
return bOut;
} }
{ BMK_runTime_t const rResult = BMK_extract_runTime(cOutcome);
while(!intermediateResultCompress.completed) { bResult.cSpeed = (srcSize * TIMELOOP_NANOSEC) / rResult.nanoSecPerRun;
intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, bResult.cSize = rResult.sumOfReturn;
nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes);
if(intermediateResultCompress.result.error) {
results.error = intermediateResultCompress.result.error;
BMK_freeTimeState(timeStateCompress);
BMK_freeTimeState(timeStateDecompress);
return results;
}
results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultCompress.result.result.nanoSecPerRun;
results.result.cSize = intermediateResultCompress.result.result.sumOfReturn;
}
while(!intermediateResultDecompress.completed) {
intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
nbBlocks, (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL);
if(intermediateResultDecompress.result.error) {
results.error = intermediateResultDecompress.result.error;
BMK_freeTimeState(timeStateCompress);
BMK_freeTimeState(timeStateDecompress);
return results;
}
results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultDecompress.result.result.nanoSecPerRun;
}
BMK_freeTimeState(timeStateCompress);
BMK_freeTimeState(timeStateDecompress);
} else { /* iterMode; */
if(mode != BMK_decodeOnly) {
BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep,
nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds);
if(compressionResults.error) {
results.error = compressionResults.error;
return results;
}
if(compressionResults.result.nanoSecPerRun == 0) {
results.result.cSpeed = 0;
} else {
results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
}
results.result.cSize = compressionResults.result.sumOfReturn;
}
if(mode != BMK_compressOnly) {
BMK_customReturn_t decompressionResults;
decompressionResults = BMK_benchFunction(
&local_defaultDecompress, (void*)(dctx),
&local_initDCtx, (void*)&dctxprep, nbBlocks,
(const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL,
nbSeconds);
if(decompressionResults.error) {
results.error = decompressionResults.error;
return results;
}
if(decompressionResults.result.nanoSecPerRun == 0) {
results.result.dSpeed = 0;
} else {
results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
}
} }
compressionCompleted = BMK_isCompleted_TimedFn(timeStateCompress);
} }
while (!decompressionCompleted) {
BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress,
&local_defaultDecompress, dctx,
&local_initDCtx, &dctxprep,
nbBlocks,
(const void* const*)dstPtrs, dstSizes,
resPtrs, resSizes,
NULL);
if (!BMK_isSuccessful_runOutcome(dOutcome)) {
BMK_benchOutcome_t bOut;
memset(&bOut, 0, sizeof(bOut));
bOut.tag = 1; /* should rather be a function or a constant */
BMK_freeTimedFnState(timeStateCompress);
BMK_freeTimedFnState(timeStateDecompress);
return bOut;
}
{ BMK_runTime_t const rResult = BMK_extract_runTime(dOutcome);
bResult.dSpeed = (srcSize * TIMELOOP_NANOSEC) / rResult.nanoSecPerRun;
}
decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress);
}
BMK_freeTimedFnState(timeStateCompress);
BMK_freeTimedFnState(timeStateDecompress);
} }
/* Bench */ /* Bench */
results.result.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx); bResult.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx);
return results;
{ BMK_benchOutcome_t bOut;
bOut.tag = 0;
bOut.internal_never_use_directly = bResult; /* should be a function */
return bOut;
}
} }
static int BMK_benchParam(BMK_result_t* resultPtr, static int BMK_benchParam ( BMK_benchResult_t* resultPtr,
const buffers_t buf, const contexts_t ctx, buffers_t buf, contexts_t ctx,
const paramValues_t cParams) { paramValues_t cParams)
BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); {
*resultPtr = res.result; BMK_benchOutcome_t const outcome = BMK_benchMemInvertible(buf, ctx,
return res.error; BASE_CLEVEL, &cParams,
BMK_both, 3);
int const success = BMK_isSuccessful_benchOutcome(outcome);
if (!success) return 1;
*resultPtr = BMK_extract_benchResult(outcome);
return 0;
} }
#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \ #define CBENCHMARK(conditional, resultvar, tmpret, mode, sec) { \
if(conditional) { \ if(conditional) { \
BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \ BMK_benchOutcome_t const outcome = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, sec); \
if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \ if (!BMK_isSuccessful_benchOutcome(outcome)) { \
if(mode != BMK_decodeOnly) { \ DEBUGOUTPUT("Benchmarking failed\n"); \
resultvar.cSpeed = tmpret.result.cSpeed; \ return ERROR_RESULT; \
resultvar.cSize = tmpret.result.cSize; \
resultvar.cMem = tmpret.result.cMem; \
} \ } \
if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \ { BMK_benchResult_t const tmpResult = BMK_extract_benchResult(outcome); \
} \ if (mode != BMK_decodeOnly) { \
resultvar.cSpeed = tmpResult.cSpeed; \
resultvar.cSize = tmpResult.cSize; \
resultvar.cMem = tmpResult.cMem; \
} \
if (mode != BMK_compressOnly) { resultvar.dSpeed = tmpResult.dSpeed; } \
} } \
} }
/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
#define VARIANCE 1.2 #define VARIANCE 1.2
static int allBench(BMK_result_t* resultPtr, static int allBench(BMK_benchResult_t* resultPtr,
const buffers_t buf, const contexts_t ctx, const buffers_t buf, const contexts_t ctx,
const paramValues_t cParams, const paramValues_t cParams,
const constraint_t target, const constraint_t target,
BMK_result_t* winnerResult, int feas) { BMK_benchResult_t* winnerResult, int feas)
BMK_result_t resultMax, benchres; {
BMK_benchResult_t benchres;
U64 loopDurationC = 0, loopDurationD = 0; U64 loopDurationC = 0, loopDurationD = 0;
double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
double winnerRS; double winnerRS;
/* initial benchmarking, gives exact ratio and memory, warms up future runs */ /* initial benchmarking, gives exact ratio and memory, warms up future runs */
CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1); CBENCHMARK(1, benchres, tmp, BMK_both, 2);
winnerRS = resultScore(*winnerResult, buf.srcSize, target); winnerRS = resultScore(*winnerResult, buf.srcSize, target);
DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
@@ -1557,12 +1564,12 @@ static int allBench(BMK_result_t* resultPtr,
/* calculate uncertainty in compression / decompression runs */ /* calculate uncertainty in compression / decompression runs */
if(benchres.cSpeed) { if(benchres.cSpeed) {
loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); loopDurationC = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
} }
if(benchres.dSpeed) { if(benchres.dSpeed) {
loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); loopDurationD = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
} }
@@ -1571,27 +1578,25 @@ static int allBench(BMK_result_t* resultPtr,
return WORSE_RESULT; return WORSE_RESULT;
} }
/* second run, if first run is too short, gives approximate cSpeed + dSpeed */ /* ensure all measurements last a minimum time, to reduce measurement errors */
CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1); assert(loopDurationC >= TIMELOOP_NANOSEC / 10);
CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1); assert(loopDurationD >= TIMELOOP_NANOSEC / 10);
*resultPtr = benchres; *resultPtr = benchres;
/* optimistic assumption of benchres */ /* optimistic assumption of benchres */
resultMax = benchres; { BMK_benchResult_t resultMax = benchres;
resultMax.cSpeed *= uncertaintyConstantC * VARIANCE; resultMax.cSpeed *= uncertaintyConstantC * VARIANCE;
resultMax.dSpeed *= uncertaintyConstantD * VARIANCE; resultMax.dSpeed *= uncertaintyConstantD * VARIANCE;
/* disregard infeasible results in feas mode */ /* disregard infeasible results in feas mode */
/* disregard if resultMax < winner in infeas mode */ /* disregard if resultMax < winner in infeas mode */
if((feas && !feasible(resultMax, target)) || if((feas && !feasible(resultMax, target)) ||
(!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
return WORSE_RESULT; return WORSE_RESULT;
}
} }
CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1);
CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1);
*resultPtr = benchres; *resultPtr = benchres;
/* compare by resultScore when in infeas */ /* compare by resultScore when in infeas */
@@ -1604,13 +1609,14 @@ static int allBench(BMK_result_t* resultPtr,
} }
} }
#define INFEASIBLE_THRESHOLD 200 #define INFEASIBLE_THRESHOLD 200
/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
static int benchMemo(BMK_result_t* resultPtr, static int benchMemo(BMK_benchResult_t* resultPtr,
const buffers_t buf, const contexts_t ctx, const buffers_t buf, const contexts_t ctx,
const paramValues_t cParams, const paramValues_t cParams,
const constraint_t target, const constraint_t target,
BMK_result_t* winnerResult, memoTable_t* const memoTableArray, BMK_benchResult_t* winnerResult, memoTable_t* const memoTableArray,
const int feas) { const int feas) {
static int bmcount = 0; static int bmcount = 0;
int res; int res;
@@ -1631,6 +1637,7 @@ static int benchMemo(BMK_result_t* resultPtr,
return res; return res;
} }
typedef struct { typedef struct {
U64 cSpeed_min; U64 cSpeed_min;
U64 dSpeed_min; U64 dSpeed_min;
@@ -1662,7 +1669,7 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
const buffers_t buf, const contexts_t ctx) const buffers_t buf, const contexts_t ctx)
{ {
BMK_result_t testResult; BMK_benchResult_t testResult;
int better = 0; int better = 0;
int cLevel; int cLevel;
@@ -1729,16 +1736,16 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
/* too large compression speed difference for the compression benefit */ /* too large compression speed difference for the compression benefit */
if (W_ratio > O_ratio) if (W_ratio > O_ratio)
DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
W_ratio, (double)testResult.cSpeed / (1 MB), W_ratio, (double)testResult.cSpeed / MB_UNIT,
O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel); O_ratio, (double)winners[cLevel].result.cSpeed / MB_UNIT, cLevel);
continue; continue;
} }
if (W_DSpeed_note < O_DSpeed_note ) { if (W_DSpeed_note < O_DSpeed_note ) {
/* too large decompression speed difference for the compression benefit */ /* too large decompression speed difference for the compression benefit */
if (W_ratio > O_ratio) if (W_ratio > O_ratio)
DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
W_ratio, (double)testResult.dSpeed / (1 MB), W_ratio, (double)testResult.dSpeed / MB_UNIT,
O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel); O_ratio, (double)winners[cLevel].result.dSpeed / MB_UNIT, cLevel);
continue; continue;
} }
@@ -1828,11 +1835,11 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
if (g_target) { if (g_target) {
BMK_init_level_constraints(g_target * (1 MB)); BMK_init_level_constraints(g_target * MB_UNIT);
} else { } else {
/* baseline config for level 1 */ /* baseline config for level 1 */
paramValues_t const l1params = cParamsToPVals(ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize)); paramValues_t const l1params = cParamsToPVals(ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize));
BMK_result_t testResult; BMK_benchResult_t testResult;
BMK_benchParam(&testResult, buf, ctx, l1params); BMK_benchParam(&testResult, buf, ctx, l1params);
BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
} }
@@ -1861,15 +1868,16 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
fclose(f); fclose(f);
} }
/*-************************************ /*-************************************
* Single Benchmark Functions * Single Benchmark Functions
**************************************/ **************************************/
static int benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) { static int benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) {
BMK_result_t testResult; BMK_benchResult_t testResult;
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
if(BMK_benchParam(&testResult, buf, ctx, g_params)) { if (BMK_benchParam(&testResult, buf, ctx, g_params)) {
DISPLAY("Error during benchmarking\n"); DISPLAY("Error during benchmarking\n");
return 1; return 1;
} }
@@ -1883,7 +1891,7 @@ static int benchSample(double compressibility, int cLevel)
{ {
const char* const name = "Sample 10MB"; const char* const name = "Sample 10MB";
size_t const benchedSize = 10 MB; size_t const benchedSize = 10 MB;
void* srcBuffer = malloc(benchedSize); void* const srcBuffer = malloc(benchedSize);
int ret = 0; int ret = 0;
buffers_t buf; buffers_t buf;
@@ -1927,31 +1935,32 @@ static int benchSample(double compressibility, int cLevel)
/* benchFiles() : /* benchFiles() :
* note: while this function takes a table of filenames, * note: while this function takes a table of filenames,
* in practice, only the first filename will be used */ * in practice, only the first filename will be used */
int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, const int cLevel) int benchFiles(const char** fileNamesTable, int nbFiles,
const char* dictFileName, int cLevel)
{ {
buffers_t buf; buffers_t buf;
contexts_t ctx; contexts_t ctx;
int ret = 0; int ret = 0;
if(createBuffers(&buf, fileNamesTable, nbFiles)) { if (createBuffers(&buf, fileNamesTable, nbFiles)) {
DISPLAY("unable to load files\n"); DISPLAY("unable to load files\n");
return 1; return 1;
} }
if(createContexts(&ctx, dictFileName)) { if (createContexts(&ctx, dictFileName)) {
DISPLAY("unable to load dictionary\n"); DISPLAY("unable to load dictionary\n");
freeBuffers(buf); freeBuffers(buf);
return 2; return 2;
} }
DISPLAY("\r%79s\r", ""); DISPLAY("\r%79s\r", "");
if(nbFiles == 1) { if (nbFiles == 1) {
DISPLAY("using %s : \n", fileNamesTable[0]); DISPLAY("using %s : \n", fileNamesTable[0]);
} else { } else {
DISPLAY("using %d Files : \n", nbFiles); DISPLAY("using %d Files : \n", nbFiles);
} }
if(g_singleRun) { if (g_singleRun) {
ret = benchOnce(buf, ctx, cLevel); ret = benchOnce(buf, ctx, cLevel);
} else { } else {
BMK_benchFullTable(buf, ctx); BMK_benchFullTable(buf, ctx);
@@ -1986,7 +1995,8 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
static winnerInfo_t climbOnce(const constraint_t target, static winnerInfo_t climbOnce(const constraint_t target,
memoTable_t* mtAll, memoTable_t* mtAll,
const buffers_t buf, const contexts_t ctx, const buffers_t buf, const contexts_t ctx,
const paramValues_t init) { const paramValues_t init)
{
/* /*
* cparam - currently considered 'center' * cparam - currently considered 'center'
* candidate - params to benchmark/results * candidate - params to benchmark/results
@@ -2000,11 +2010,9 @@ static winnerInfo_t climbOnce(const constraint_t target,
winnerInfo = initWinnerInfo(init); winnerInfo = initWinnerInfo(init);
candidateInfo = winnerInfo; candidateInfo = winnerInfo;
{ { winnerInfo_t bestFeasible1 = initWinnerInfo(cparam);
winnerInfo_t bestFeasible1 = initWinnerInfo(cparam);
DEBUGOUTPUT("Climb Part 1\n"); DEBUGOUTPUT("Climb Part 1\n");
while(better) { while(better) {
int offset; int offset;
size_t i, dist; size_t i, dist;
const size_t varLen = mtAll[cparam.vals[strt_ind]].varLen; const size_t varLen = mtAll[cparam.vals[strt_ind]].varLen;
@@ -2013,8 +2021,8 @@ static winnerInfo_t climbOnce(const constraint_t target,
cparam = winnerInfo.params; cparam = winnerInfo.params;
candidateInfo.params = cparam; candidateInfo.params = cparam;
/* all dist-1 candidates */ /* all dist-1 candidates */
for(i = 0; i < varLen; i++) { for (i = 0; i < varLen; i++) {
for(offset = -1; offset <= 1; offset += 2) { for (offset = -1; offset <= 1; offset += 2) {
CHECKTIME(winnerInfo); CHECKTIME(winnerInfo);
candidateInfo.params = cparam; candidateInfo.params = cparam;
paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params); paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params);
@@ -2033,7 +2041,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
} }
} }
} }
} } /* for (i = 0; i < varLen; i++) */
if(better) { if(better) {
continue; continue;
@@ -2047,27 +2055,28 @@ static winnerInfo_t climbOnce(const constraint_t target,
/* param error checking already done here */ /* param error checking already done here */
paramVariation(&candidateInfo.params, mtAll, (U32)dist); paramVariation(&candidateInfo.params, mtAll, (U32)dist);
res = benchMemo(&candidateInfo.result, buf, ctx, res = benchMemo(&candidateInfo.result,
sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas); buf, ctx,
sanitizeParams(candidateInfo.params), target,
&winnerInfo.result, mtAll, feas);
DEBUGOUTPUT("Res: %d\n", res); DEBUGOUTPUT("Res: %d\n", res);
if(res == BETTER_RESULT) { /* synonymous with better in this case*/ if (res == BETTER_RESULT) { /* synonymous with better in this case*/
winnerInfo = candidateInfo; winnerInfo = candidateInfo;
better = 1; better = 1;
if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { if (compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
bestFeasible1 = winnerInfo; bestFeasible1 = winnerInfo;
} }
break; break;
} }
} }
if(better) {
if (better) {
break; break;
} }
} } /* for(dist = 2; dist < varLen + 2; dist++) */
if(!better) { /* infeas -> feas -> stop */
if(feas) { return winnerInfo; }
if (!better) { /* infeas -> feas -> stop */
if (feas) return winnerInfo;
feas = 1; feas = 1;
better = 1; better = 1;
winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */ winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */
@@ -2203,14 +2212,14 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
allMT = createMemoTableArray(paramTarget, varArray, varLen, memoTableLog); allMT = createMemoTableArray(paramTarget, varArray, varLen, memoTableLog);
if(!allMT) { if (!allMT) {
DISPLAY("MemoTable Init Error\n"); DISPLAY("MemoTable Init Error\n");
ret = 2; ret = 2;
goto _cleanUp; goto _cleanUp;
} }
/* default strictnesses */ /* default strictnesses */
if(g_strictness == PARAM_UNSET) { if (g_strictness == PARAM_UNSET) {
if(g_optmode) { if(g_optmode) {
g_strictness = 100; g_strictness = 100;
} else { } else {
@@ -2225,7 +2234,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
} }
/* use level'ing mode instead of normal target mode */ /* use level'ing mode instead of normal target mode */
if(g_optmode) { if (g_optmode) {
winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize)); winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize));
if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
ret = 3; ret = 3;
@@ -2244,10 +2253,10 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
} }
/* Don't want it to return anything worse than the best known result */ /* Don't want it to return anything worse than the best known result */
if(g_singleRun) { if (g_singleRun) {
BMK_result_t res; BMK_benchResult_t res;
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
if(BMK_benchParam(&res, buf, ctx, g_params)) { if (BMK_benchParam(&res, buf, ctx, g_params)) {
ret = 45; ret = 45;
goto _cleanUp; goto _cleanUp;
} }
@@ -2272,8 +2281,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
DISPLAYLEVEL(2, "\n"); DISPLAYLEVEL(2, "\n");
findClockGranularity(); findClockGranularity();
{ { paramValues_t CParams;
paramValues_t CParams;
/* find best solution from default params */ /* find best solution from default params */
{ {
@@ -2281,7 +2289,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
DEBUGOUTPUT("Strategy Selection\n"); DEBUGOUTPUT("Strategy Selection\n");
if(paramTarget.vals[strt_ind] == PARAM_UNSET) { if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
BMK_result_t candidate; BMK_benchResult_t candidate;
int i; int i;
for (i=1; i<=maxSeeds; i++) { for (i=1; i<=maxSeeds; i++) {
int ec; int ec;