lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

bench.c: support advanced compression parameters

Browse Source
This commit is contained in:
Przemyslaw Skibinski
2016-12-13 13:03:41 +01:00
parent 334b8962b0
commit 897b8bb5eb
3 changed files with 31 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
programs/bench.c
View File

@ -125,7 +125,8 @@ typedef struct {
static int BMK_benchMem(const void* srcBuffer, size_t srcSize, static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel, const char* displayName, int cLevel,
const size_t* fileSizes, U32 nbFiles, const size_t* fileSizes, U32 nbFiles,
const void* dictBuffer, size_t dictBufferSize) const void* dictBuffer, size_t dictBufferSize,
ZSTD_compressionParameters *comprParams)
{ {
size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; size_t const blockSize = ((g_blockSize>=32 && !g_decodeOnly) ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles)); size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
@ -231,12 +232,19 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_getTime(&clockStart); UTIL_getTime(&clockStart);
if (!cCompleted) { /* still some time to do compression tests */ if (!cCompleted) { /* still some time to do compression tests */
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL }; ZSTD_customMem const cmem = { NULL, NULL, NULL };
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0; U32 nbLoops = 0;
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
do { do {
U32 blockNb; U32 blockNb;
size_t rSize; size_t rSize;
@ -247,9 +255,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cdict); cdict);
} else { } else {
rSize = ZSTD_compressCCtx (ctx, rSize = ZSTD_compress_advanced (ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel); blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams);
} }
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize)); if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].cSize = rSize; blockTable[blockNb].cSize = rSize;
@ -387,7 +395,8 @@ static size_t BMK_findMaxMem(U64 requiredMem)
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast, const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize) const void* dictBuffer, size_t dictBufferSize,
ZSTD_compressionParameters *compressionParams)
{ {
int l; int l;
@ -406,7 +415,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
BMK_benchMem(srcBuffer, benchedSize, BMK_benchMem(srcBuffer, benchedSize,
displayName, l, displayName, l,
fileSizes, nbFiles, fileSizes, nbFiles,
dictBuffer, dictBufferSize); dictBuffer, dictBufferSize, compressionParams);
} }
} }
@ -443,8 +452,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (totalSize == 0) EXM_THROW(12, "no data to bench"); if (totalSize == 0) EXM_THROW(12, "no data to bench");
} }
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
const char* dictFileName, int cLevel, int cLevelLast) int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams)
{ {
void* srcBuffer; void* srcBuffer;
size_t benchedSize; size_t benchedSize;
@ -483,7 +492,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
BMK_benchCLevel(srcBuffer, benchedSize, BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast, displayName, cLevel, cLevelLast,
fileSizes, nbFiles, fileSizes, nbFiles,
dictBuffer, dictBufferSize); dictBuffer, dictBufferSize, compressionParams);
} }
/* clean up */ /* clean up */
@ -493,7 +502,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
} }
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams)
{ {
char name[20] = {0}; char name[20] = {0};
size_t benchedSize = 10000000; size_t benchedSize = 10000000;
@ -507,15 +516,15 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
/* Bench */ /* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);
/* clean up */ /* clean up */
free(srcBuffer); free(srcBuffer);
} }
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
const char* dictFileName, int cLevel, int cLevelLast) int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams)
{ {
double const compressibility = (double)g_compressibilityDefault / 100; double const compressibility = (double)g_compressibilityDefault / 100;
@ -525,8 +534,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (nbFiles == 0) if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility); BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
else else
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
return 0; return 0;
} }

6
programs/bench.h
View File

@ -12,9 +12,11 @@
#define BENCH_H_121279284357 #define BENCH_H_121279284357
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "zstd.h" /* ZSTD_compressionParameters */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
const char* dictFileName, int cLevel, int cLevelLast); int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams);
/* Set Parameters */ /* Set Parameters */
void BMK_SetNbSeconds(unsigned nbLoops); void BMK_SetNbSeconds(unsigned nbLoops);

12
programs/zstdcli.c
View File

@ -214,7 +214,6 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params) static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
{ {
for ( ; ;) { for ( ; ;) {
DISPLAY("arg=%s\n", stringPtr);
if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
@ -226,13 +225,8 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
} }
if (stringPtr[0] != 0) return 0; /* check the end of string */ if (stringPtr[0] != 0) return 0; /* check the end of string */
DISPLAYLEVEL(4, "windowLog=%d\n", params->windowLog); DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
DISPLAYLEVEL(4, "chainLog=%d\n", params->chainLog); DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy);
DISPLAYLEVEL(4, "hashLog=%d\n", params->hashLog);
DISPLAYLEVEL(4, "searchLog=%d\n", params->searchLog);
DISPLAYLEVEL(4, "searchLength=%d\n", params->searchLength);
DISPLAYLEVEL(4, "targetLength=%d\n", params->targetLength);
DISPLAYLEVEL(4, "strategy=%d\n", params->strategy);
return 1; return 1;
} }
@ -523,7 +517,7 @@ int main(int argCount, const char* argv[])
if (operation==zom_bench) { if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
BMK_setNotificationLevel(displayLevel); BMK_setNotificationLevel(displayLevel);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast); BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
#endif #endif
goto _end; goto _end;
} }