lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

Modify benchmark to only load sources once

Browse Source 
After a regrettable update,
the benchmark module ended up reloading sources for every compression level.

While the delay itself is likely torelable,
the main issue is that the `--quiet` mode now also displays a loading summary between each compression line.
This wasn't the original intention, which is to produce a compact view of all compressions.

This is fixed in this version,
where sources are loaded only once, for all compression levels,
and loading summary is only displayed once.
This commit is contained in:
Yann Collet
2024-10-22 02:18:48 -07:00
parent b880f20d52
commit 0079d515b1
3 changed files with 57 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
programs/benchzstd.c
View File

@ -928,12 +928,13 @@ BMK_benchOutcome_t BMK_benchMem(
&adv); &adv);
} }
static BMK_benchOutcome_t BMK_benchCLevel( /* @return: 0 on success, !0 if error */
static int BMK_benchCLevels(
const void* srcBuffer, const void* srcBuffer,
size_t benchedSize, size_t benchedSize,
const size_t* fileSizes, const size_t* fileSizes,
unsigned nbFiles, unsigned nbFiles,
int cLevel, int startCLevel, int endCLevel,
const ZSTD_compressionParameters* comprParams, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, const void* dictBuffer,
size_t dictBufferSize, size_t dictBufferSize,
@ -941,12 +942,22 @@ static BMK_benchOutcome_t BMK_benchCLevel(
const char* displayName, const char* displayName,
BMK_advancedParams_t const* const adv) BMK_advancedParams_t const* const adv)
{ {
int level;
const char* pch = strrchr(displayName, '\\'); /* Windows */ const char* pch = strrchr(displayName, '\\'); /* Windows */
if (!pch) if (!pch)
pch = strrchr(displayName, '/'); /* Linux */ pch = strrchr(displayName, '/'); /* Linux */
if (pch) if (pch)
displayName = pch + 1; displayName = pch + 1;
if (endCLevel > ZSTD_maxCLevel()) {
DISPLAYLEVEL(1, "Invalid Compression Level \n");
return 15;
}
if (endCLevel < startCLevel) {
DISPLAYLEVEL(1, "Invalid Compression Level Range \n");
return 15;
}
if (adv->realTime) { if (adv->realTime) {
DISPLAYLEVEL(2, "Note : switching to real-time priority \n"); DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
SET_REALTIME_PRIORITY; SET_REALTIME_PRIORITY;
@ -960,25 +971,29 @@ static BMK_benchOutcome_t BMK_benchCLevel(
adv->nbSeconds, adv->nbSeconds,
(unsigned)(adv->blockSize >> 10)); (unsigned)(adv->blockSize >> 10));
return BMK_benchMemAdvanced( for (level = startCLevel; level <= endCLevel; level++) {
BMK_benchOutcome_t res = BMK_benchMemAdvanced(
srcBuffer, srcBuffer,
benchedSize, benchedSize,
NULL, NULL,
0, 0,
fileSizes, fileSizes,
nbFiles, nbFiles,
cLevel, level,
comprParams, comprParams,
dictBuffer, dictBuffer,
dictBufferSize, dictBufferSize,
displayLevel, displayLevel,
displayName, displayName,
adv); adv);
if (!BMK_isSuccessful_benchOutcome(res)) return 1;
}
return 0;
} }
int BMK_syntheticTest( int BMK_syntheticTest(
int cLevel,
double compressibility, double compressibility,
int startingCLevel, int endCLevel,
const ZSTD_compressionParameters* compressionParams, const ZSTD_compressionParameters* compressionParams,
int displayLevel, int displayLevel,
const BMK_advancedParams_t* adv) const BMK_advancedParams_t* adv)
@ -986,18 +1001,11 @@ int BMK_syntheticTest(
char nameBuff[20] = { 0 }; char nameBuff[20] = { 0 };
const char* name = nameBuff; const char* name = nameBuff;
size_t const benchedSize = adv->blockSize ? adv->blockSize : 10000000; size_t const benchedSize = adv->blockSize ? adv->blockSize : 10000000;
void* srcBuffer;
BMK_benchOutcome_t res;
if (cLevel > ZSTD_maxCLevel()) {
DISPLAYLEVEL(1, "Invalid Compression Level");
return 15;
}
/* Memory allocation */ /* Memory allocation */
srcBuffer = malloc(benchedSize); void* const srcBuffer = malloc(benchedSize);
if (!srcBuffer) { if (!srcBuffer) {
DISPLAYLEVEL(1, "allocation error : not enough memory"); DISPLAYLEVEL(1, "allocation error : not enough memory \n");
return 16; return 16;
} }
@ -1015,23 +1023,21 @@ int BMK_syntheticTest(
} }
/* Bench */ /* Bench */
res = BMK_benchCLevel( { int res = BMK_benchCLevels(
srcBuffer, srcBuffer,
benchedSize, benchedSize,
&benchedSize /* ? */, &benchedSize,
1 /* ? */, 1,
cLevel, startingCLevel, endCLevel,
compressionParams, compressionParams,
NULL, NULL,
0, /* dictionary */ 0, /* dictionary */
displayLevel, displayLevel,
name, name,
adv); adv);
free(srcBuffer);
/* clean up */ return res;
free(srcBuffer); }
return !BMK_isSuccessful_benchOutcome(res);
} }
static size_t BMK_findMaxMem(U64 requiredMem) static size_t BMK_findMaxMem(U64 requiredMem)
@ -1120,7 +1126,7 @@ int BMK_benchFilesAdvanced(
const char* const* fileNamesTable, const char* const* fileNamesTable,
unsigned nbFiles, unsigned nbFiles,
const char* dictFileName, const char* dictFileName,
int cLevel, int startCLevel, int endCLevel,
const ZSTD_compressionParameters* compressionParams, const ZSTD_compressionParameters* compressionParams,
int displayLevel, int displayLevel,
const BMK_advancedParams_t* adv) const BMK_advancedParams_t* adv)
@ -1130,7 +1136,7 @@ int BMK_benchFilesAdvanced(
void* dictBuffer = NULL; void* dictBuffer = NULL;
size_t dictBufferSize = 0; size_t dictBufferSize = 0;
size_t* fileSizes = NULL; size_t* fileSizes = NULL;
BMK_benchOutcome_t res; int res = 1;
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
if (!nbFiles) { if (!nbFiles) {
@ -1138,7 +1144,7 @@ int BMK_benchFilesAdvanced(
return 13; return 13;
} }
if (cLevel > ZSTD_maxCLevel()) { if (endCLevel > ZSTD_maxCLevel()) {
DISPLAYLEVEL(1, "Invalid Compression Level"); DISPLAYLEVEL(1, "Invalid Compression Level");
return 14; return 14;
} }
@ -1192,7 +1198,6 @@ int BMK_benchFilesAdvanced(
1 /*?*/, 1 /*?*/,
displayLevel); displayLevel);
if (errorCode) { if (errorCode) {
res = BMK_benchOutcome_error();
goto _cleanUp; goto _cleanUp;
} }
} }
@ -1224,7 +1229,6 @@ int BMK_benchFilesAdvanced(
nbFiles, nbFiles,
displayLevel); displayLevel);
if (errorCode) { if (errorCode) {
res = BMK_benchOutcome_error();
goto _cleanUp; goto _cleanUp;
} }
} }
@ -1233,15 +1237,14 @@ int BMK_benchFilesAdvanced(
{ {
char mfName[20] = { 0 }; char mfName[20] = { 0 };
formatString_u(mfName, sizeof(mfName), " %u files", nbFiles); formatString_u(mfName, sizeof(mfName), " %u files", nbFiles);
{ { const char* const displayName =
const char* const displayName =
(nbFiles > 1) ? mfName : fileNamesTable[0]; (nbFiles > 1) ? mfName : fileNamesTable[0];
res = BMK_benchCLevel( res = BMK_benchCLevels(
srcBuffer, srcBuffer,
benchedSize, benchedSize,
fileSizes, fileSizes,
nbFiles, nbFiles,
cLevel, startCLevel, endCLevel,
compressionParams, compressionParams,
dictBuffer, dictBuffer,
dictBufferSize, dictBufferSize,
@ -1255,7 +1258,7 @@ _cleanUp:
free(srcBuffer); free(srcBuffer);
free(dictBuffer); free(dictBuffer);
free(fileSizes); free(fileSizes);
return !BMK_isSuccessful_benchOutcome(res); return res;
} }
int BMK_benchFiles( int BMK_benchFiles(
@ -1271,7 +1274,7 @@ int BMK_benchFiles(
fileNamesTable, fileNamesTable,
nbFiles, nbFiles,
dictFileName, dictFileName,
cLevel, cLevel, cLevel,
compressionParams, compressionParams,
displayLevel, displayLevel,
&adv); &adv);

10
programs/benchzstd.h
View File

@ -122,12 +122,13 @@ BMK_advancedParams_t BMK_initAdvancedParams(void);
int BMK_benchFilesAdvanced( int BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles, const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName, const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams, int startCLevel, int endCLevel,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv); int displayLevel, const BMK_advancedParams_t* adv);
/*! BMK_syntheticTest() -- called from zstdcli */ /*! BMK_syntheticTest() -- called from zstdcli */
/* Generates a sample with datagen, using compressibility argument */ /* Generates a sample with datagen, using @compressibility argument
/* @cLevel - compression level to benchmark, errors if invalid * @cLevel - compression level to benchmark, errors if invalid
* @compressibility - determines compressibility of sample, range [0.0 - 1.0] * @compressibility - determines compressibility of sample, range [0.0 - 1.0]
* if @compressibility < 0.0, uses the lorem ipsum generator * if @compressibility < 0.0, uses the lorem ipsum generator
* @compressionParams - basic compression Parameters * @compressionParams - basic compression Parameters
@ -135,7 +136,8 @@ int BMK_benchFilesAdvanced(
* @adv - see advanced_Params_t * @adv - see advanced_Params_t
* @return: 0 on success, !0 on error * @return: 0 on success, !0 on error
*/ */
int BMK_syntheticTest(int cLevel, double compressibility, int BMK_syntheticTest(double compressibility,
int startingCLevel, int endCLevel,
const ZSTD_compressionParameters* compressionParams, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv); int displayLevel, const BMK_advancedParams_t* adv);

16
programs/zstdcli.c
View File

@ -1400,19 +1400,15 @@ int main(int argCount, const char* argv[])
if(separateFiles) { if(separateFiles) {
unsigned i; unsigned i;
for(i = 0; i < filenames->tableSize; i++) { for(i = 0; i < filenames->tableSize; i++) {
int c;
DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]); DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
for(c = cLevel; c <= cLevelLast; c++) { operationResult = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel, &benchParams);
operationResult = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams); }
} }
} else { } else {
for(; cLevel <= cLevelLast; cLevel++) { operationResult = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel, &benchParams);
operationResult = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams); }
} }
} else { } else {
for(; cLevel <= cLevelLast; cLevel++) { operationResult = BMK_syntheticTest(compressibility, cLevel, cLevelLast, &compressionParams, g_displayLevel, &benchParams);
operationResult = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams); }
} }
#else #else
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility; (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;