1
0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00

Avoid snprintf() in Preparing Human-Readable Sizes; Improve Formatting

This produces the following formatting:

   Size    | `zstd` | `ls -lh`
---------- | ------ | --------
1          | 1      | 1
12         | 12     | 12
123        | 123    | 123
1234       | 1.21K  | 1.3K
12345      | 12.1K  | 13K
123456     | 121K   | 121K
1234567    | 1.18M  | 1.2M
12345678   | 11.8M  | 12M
123456789  | 118M   | 118M
1234567890 | 1.15G  | 1.2G
999        | 999    | 999
1000       | 1000   | 1000
1001       | 1001   | 1001
1023       | 1023   | 1023
1024       | 1.000K | 1.0K
1025       | 1.00K  | 1.1K
999999     | 977K   | 977K
1000000    | 977K   | 977K
1000001    | 977K   | 977K
1023999    | 1000K  | 1000K
1024000    | 1000K  | 1000K
1024001    | 1000K  | 1001K
1048575    | 1024K  | 1.0M
1048576    | 1.000M | 1.0M
1048577    | 1.00M  | 1.1M

This was produced with the following invocation:

```
for N in 1 12 123 1234 12345 123456 1234567 12345678 123456789 1234567890 999 1000 1001 1023 1024 1025 999999 1000000 1000001 1023999 1024000 1024001 1048575 1048576 1048577; do
  head -c $N /dev/urandom > r$N
done
./zstd -i1 -b1 -S r1 r12 r123 r1234 r12345 r123456 r1234567 r12345678 r123456789 r1234567890 r999 r1000 r1001 r1023 r1024 r1025 r999999 r1000000 r1000001 r1023999 r1024000 r1024001 r1048575 r1048576 r1048577
```
This commit is contained in:
W. Felix Handte
2021-06-09 13:05:44 -04:00
parent 8e0a9695d7
commit bbb81c8801
4 changed files with 73 additions and 59 deletions

View File

@ -388,13 +388,13 @@ BMK_benchMemAdvancedNoAlloc(
# define NB_MARKS 4 # define NB_MARKS 4
const char* marks[NB_MARKS] = { " |", " /", " =", " \\" }; const char* marks[NB_MARKS] = { " |", " /", " =", " \\" };
U32 markNb = 0; U32 markNb = 0;
char inputSizeStr[8] = "";
char outputSizeStr[8] = "";
int compressionCompleted = (adv->mode == BMK_decodeOnly); int compressionCompleted = (adv->mode == BMK_decodeOnly);
int decompressionCompleted = (adv->mode == BMK_compressOnly); int decompressionCompleted = (adv->mode == BMK_compressOnly);
BMK_benchParams_t cbp, dbp; BMK_benchParams_t cbp, dbp;
BMK_initCCtxArgs cctxprep; BMK_initCCtxArgs cctxprep;
BMK_initDCtxArgs dctxprep; BMK_initDCtxArgs dctxprep;
UTIL_HumanReadableSize_t hr_isize;
UTIL_HumanReadableSize_t hr_osize;
cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */
cbp.benchPayload = cctx; cbp.benchPayload = cctx;
@ -431,10 +431,10 @@ BMK_benchMemAdvancedNoAlloc(
dctxprep.dictBuffer = dictBuffer; dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize; dctxprep.dictBufferSize = dictBufferSize;
humanSize((unsigned)srcSize, inputSizeStr); hr_isize = UTIL_makeHumanReadableSize((U64) srcSize);
DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */
DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> \r", marks[markNb], displayName, inputSizeStr); DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> \r", marks[markNb], displayName, hr_isize.precision, hr_isize.value, hr_isize.suffix);
while (!(compressionCompleted && decompressionCompleted)) { while (!(compressionCompleted && decompressionCompleted)) {
if (!compressionCompleted) { if (!compressionCompleted) {
@ -455,13 +455,11 @@ BMK_benchMemAdvancedNoAlloc(
} } } }
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
hr_osize = UTIL_makeHumanReadableSize((U64) cSize);
humanSize((unsigned)srcSize, inputSizeStr); DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f),%6.*f MB/s\r",
humanSize((unsigned)cSize, outputSizeStr);
DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> %s (%5.*f),%6.*f MB/s\r",
marks[markNb], displayName, marks[markNb], displayName,
inputSizeStr, outputSizeStr, hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
ratioAccuracy, ratio, ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT); benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT);
} }
@ -482,13 +480,11 @@ BMK_benchMemAdvancedNoAlloc(
} }
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
hr_osize = UTIL_makeHumanReadableSize((U64) cSize);
humanSize((unsigned)srcSize, inputSizeStr); DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
humanSize((unsigned)cSize, outputSizeStr);
DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> %s (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName, marks[markNb], displayName,
inputSizeStr, outputSizeStr, hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
ratioAccuracy, ratio, ratioAccuracy, ratio,
benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT,
(double)benchResult.dSpeed / MB_UNIT); (double)benchResult.dSpeed / MB_UNIT);

View File

@ -1544,9 +1544,6 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
U64 readsize = 0; U64 readsize = 0;
U64 compressedfilesize = 0; U64 compressedfilesize = 0;
U64 const fileSize = UTIL_getFileSize(srcFileName); U64 const fileSize = UTIL_getFileSize(srcFileName);
char inputSizeStr[8] = "";
char outputSizeStr[8] = "";
DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize); DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize);
/* compression format selection */ /* compression format selection */
@ -1601,13 +1598,14 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
(unsigned long long)readsize, (unsigned long long) compressedfilesize, (unsigned long long)readsize, (unsigned long long) compressedfilesize,
dstFileName); dstFileName);
} else { } else {
humanSize((unsigned long long) readsize, inputSizeStr); UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize);
humanSize((unsigned long long) compressedfilesize, outputSizeStr); UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize);
DISPLAYLEVEL(2,"%-20s :%6.2f%% (%s => %s, %s) \n", DISPLAYLEVEL(2,"%-20s :%6.2f%% (%.*f%s => %.*f%s, %s) \n",
srcFileName, srcFileName,
(double)compressedfilesize / (double)readsize * 100, (double)compressedfilesize / (double)readsize * 100,
inputSizeStr, outputSizeStr, hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
dstFileName); dstFileName);
} }
} }
@ -1841,8 +1839,6 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
{ {
int status; int status;
int error = 0; int error = 0;
char inputSizeStr[8] = "";
char outputSizeStr[8] = "";
cRess_t ress = FIO_createCResources(prefs, dictFileName, cRess_t ress = FIO_createCResources(prefs, dictFileName,
FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal), FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal),
compressionLevel, comprParams); compressionLevel, comprParams);
@ -1898,13 +1894,15 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
} }
if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) {
humanSize((unsigned long long) fCtx->totalBytesInput, inputSizeStr); UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput);
humanSize((unsigned long long) fCtx->totalBytesOutput, outputSizeStr); UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput);
DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "%3d files compressed : %.2f%% (%s => %s bytes)\n", fCtx->nbFilesProcessed, DISPLAYLEVEL(2, "%3d files compressed : %.2f%% (%.*f%s => %.*f%s bytes)\n",
fCtx->nbFilesProcessed,
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
inputSizeStr, outputSizeStr); hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
} }
FIO_freeCResources(&ress); FIO_freeCResources(&ress);

View File

@ -121,31 +121,6 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
* Functions * Functions
***************************************/ ***************************************/
/*
* Take a size in bytes and output a human readable string. Maximum
* buffer size is 8 but it's usually 7. Example: "123.4G"
*/
char* humanSize(unsigned long long size, char* str) {
if (size > 1152921504606846976L) {
snprintf(str, 7, "%.1fE", (float)size / 1152921504606846976L);
} else if (size > 1125899906842624L) {
snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L);
} else if (size > 1099511627776L) {
snprintf(str, 7, "%.1fT", (float)size / 1099511627776L);
} else if (size > 1073741824L) {
snprintf(str, 7, "%.1fG", (float)size / 1073741824L);
} else if (size > 1048576L) {
snprintf(str, 7, "%.1fM", (float)size / 1048576L);
} else if (size > 1024) {
snprintf(str, 7, "%.1fK", (float)size / 1024);
} else if (size <= 1024) {
snprintf(str, 7, "%uB", (unsigned)size);
}
return str;
}
int UTIL_stat(const char* filename, stat_t* statbuf) int UTIL_stat(const char* filename, stat_t* statbuf)
{ {
#if defined(_MSC_VER) #if defined(_MSC_VER)
@ -328,6 +303,43 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf)
return (U64)statbuf->st_size; return (U64)statbuf->st_size;
} }
UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) {
UTIL_HumanReadableSize_t hrs;
if (size >= (1L << 60)) {
hrs.value = (float)size / (1L << 60);
hrs.suffix = "E";
} else if (size >= (1L << 50)) {
hrs.value = (float)size / (1L << 50);
hrs.suffix = "P";
} else if (size >= (1L << 40)) {
hrs.value = (float)size / (1L << 40);
hrs.suffix = "T";
} else if (size >= (1L << 30)) {
hrs.value = (float)size / (1L << 30);
hrs.suffix = "G";
} else if (size >= (1L << 20)) {
hrs.value = (float)size / (1L << 20);
hrs.suffix = "M";
} else if (size >= (1L << 10)) {
hrs.value = (float)size / (1L << 10);
hrs.suffix = "K";
} else {
hrs.value = (float)size;
hrs.suffix = "";
}
if (hrs.value >= 100 || (U64)hrs.value == size) {
hrs.precision = 0;
} else if (hrs.value > 10) {
hrs.precision = 1;
} else if (hrs.value > 1) {
hrs.precision = 2;
} else {
hrs.precision = 3;
}
return hrs;
}
U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
{ {

View File

@ -122,11 +122,6 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const
#define STRDUP(s) strdup(s) #define STRDUP(s) strdup(s)
#endif #endif
/*
* Take a size in bytes and output a human readable string. Maximum
* buffer size is 8 but it's usually 7. Example: "123.4G"
*/
char* humanSize(unsigned long long size, char* str);
/** /**
* Calls platform's equivalent of stat() on filename and writes info to statbuf. * Calls platform's equivalent of stat() on filename and writes info to statbuf.
@ -176,6 +171,19 @@ int UTIL_isFIFO(const char* infilename);
U64 UTIL_getFileSize(const char* infilename); U64 UTIL_getFileSize(const char* infilename);
U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles); U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
/**
* Take a size in bytes and prepare the components to pretty-print it in a
* scaled way. The components in the returned struct should be passed in
* precision, value, suffix order to a "%.*f%s" format string.
*/
typedef struct {
float value;
int precision;
const char* suffix;
} UTIL_HumanReadableSize_t;
UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size);
int UTIL_compareStr(const void *p1, const void *p2); int UTIL_compareStr(const void *p1, const void *p2);
const char* UTIL_getFileExtension(const char* infilename); const char* UTIL_getFileExtension(const char* infilename);
void UTIL_mirrorSourceFilesDirectories(const char** fileNamesTable, unsigned int nbFiles, const char *outDirName); void UTIL_mirrorSourceFilesDirectories(const char** fileNamesTable, unsigned int nbFiles, const char *outDirName);