1
0
mirror of https://github.com/facebook/zstd.git synced 2025-07-29 11:21:22 +03:00

Refactor progress bar & summary line logic

* Centralize the logic about whether to print the progress bar or not in
  the `*_PROGRESS()` macros.
* Centralize the logc about whether to print the summary line or not in
  `FIO_shouldDisplayFileSummary()` and
  `FIO_shouldDisplayMultipleFileSummary()`.
* Make `--progress` work for non-zstd (de)compressors.
* Clean up several edge cases in compression and decompression progress
  printing along the way. E.g. wrong log level, or missing summary line.

One thing I don't like about stdout mode, which sets the display level
to 1, is that warnings aren't displayed. After this PR, we could change
stdout mode from lowering the display level, to defaulting to implied
`--no-progress`. But, I think that deserves a separate PR.
This commit is contained in:
Nick Terrell
2022-01-07 15:07:28 -08:00
committed by Nick Terrell
parent e58a39f84e
commit fbff7827fa
6 changed files with 318 additions and 52 deletions

View File

@ -249,6 +249,18 @@ struct FIO_ctx_s {
size_t totalBytesOutput;
};
static int FIO_shouldDisplayFileSummary(FIO_ctx_t const* fCtx)
{
return fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3;
}
static int FIO_shouldDisplayMultipleFileSummary(FIO_ctx_t const* fCtx)
{
int const shouldDisplay = (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1);
assert(shouldDisplay || FIO_shouldDisplayFileSummary(fCtx) || fCtx->nbFilesProcessed == 0);
return shouldDisplay;
}
/*-*************************************
* Parameters: Initialization
@ -1044,11 +1056,13 @@ FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but no
strm.avail_out = (uInt)writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
DISPLAYUPDATE_PROGRESS(
"\rRead : %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
DISPLAYUPDATE_PROGRESS(
"\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
} }
@ -1141,11 +1155,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
strm.avail_out = writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
else
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
if (ret == LZMA_STREAM_END) break;
@ -1225,11 +1239,11 @@ FIO_compressLz4Frame(cRess_t* ress,
srcFileName, LZ4F_getErrorName(outSize));
outFileSize += outSize;
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100)
} else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
DISPLAYUPDATE_PROGRESS("\rRead : %u / %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100);
}
@ -1364,32 +1378,34 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced);
/* display progress notifications */
DISPLAY_PROGRESS("\r%79s\r", ""); /* Clear out the current displayed line */
if (g_display_prefs.displayLevel >= 3) {
DISPLAYUPDATE(3, "\r(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
/* Verbose progress update */
DISPLAY_PROGRESS(
"(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ",
compressionLevel,
buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix,
consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix,
produced_hrs.precision, produced_hrs.value, produced_hrs.suffix,
cShare );
} else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) {
} else {
/* Require level 2 or forcibly displayed progress counter for summarized updates */
DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */
if (fCtx->nbFilesTotal > 1) {
size_t srcFileNameSize = strlen(srcFileName);
/* Ensure that the string we print is roughly the same size each time */
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ",
DISPLAY_PROGRESS("Compress: %u/%u files. Current: ...%s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName);
} else {
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ",
DISPLAY_PROGRESS("Compress: %u/%u files. Current: %*s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName);
}
}
DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
DISPLAY_PROGRESS("Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix);
if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
DISPLAYLEVEL(1, " ==> %2.f%%", cShare);
DISPLAY_PROGRESS("/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix);
DISPLAY_PROGRESS(" ==> %2.f%%", cShare);
DELAY_NEXT_UPDATE();
}
@ -1555,20 +1571,18 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
/* Status */
fCtx->totalBytesInput += (size_t)readsize;
fCtx->totalBytesOutput += (size_t)compressedfilesize;
DISPLAYLEVEL(2, "\r%79s\r", "");
if (g_display_prefs.displayLevel >= 2 &&
!fCtx->hasStdoutOutput &&
(g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) {
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize);
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize);
if (readsize == 0) {
DISPLAYLEVEL(2,"%-20s : (%6.*f%s => %6.*f%s, %s) \n",
DISPLAY_SUMMARY("%-20s : (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix,
dstFileName);
} else {
DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n",
DISPLAY_SUMMARY("%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
(double)compressedfilesize / (double)readsize * 100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
@ -1917,17 +1931,24 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);
}
if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) {
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput);
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput);
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n",
DISPLAY_PROGRESS("\r%79s\r", "");
if (fCtx->totalBytesInput == 0) {
DISPLAY_SUMMARY("%3d files compressed : (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
} else {
DISPLAY_SUMMARY("%3d files compressed : %.2f%% (%6.*f%4s => %6.*f%4s)\n",
fCtx->nbFilesProcessed,
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
hr_isize.precision, hr_isize.value, hr_isize.suffix,
hr_osize.precision, hr_osize.value, hr_osize.suffix);
}
}
FIO_freeCResources(&ress);
return error;
@ -2067,7 +2088,6 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
ZSTD_inBuffer inBuff = { ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 };
ZSTD_outBuffer outBuff= { writeJob->buffer, writeJob->bufferSize, 0 };
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2;
UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize);
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
@ -2085,14 +2105,15 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
size_t srcFileNameSize = strlen(srcFileName);
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ",
DISPLAYUPDATE_PROGRESS(
"\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
} else {
DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
DISPLAYUPDATE_PROGRESS("\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix);
}
} else {
DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s... ",
DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s... ",
srcFileName, hrs.precision, hrs.value, hrs.suffix);
}
@ -2307,7 +2328,7 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName)
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
filesize += decodedBytes;
hrs = UTIL_makeHumanReadableSize(filesize);
DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix);
DISPLAYUPDATE_PROGRESS("\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix);
}
if (!nextToLoad) break;
@ -2415,13 +2436,9 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
/* Final Status */
fCtx->totalBytesOutput += (size_t)filesize;
DISPLAYLEVEL(2, "\r%79s\r", "");
/* No status message in pipe mode (stdin - stdout) or multi-files mode */
if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) ||
g_display_prefs.displayLevel >= 3 ||
g_display_prefs.progressSetting == FIO_ps_always) {
DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize);
}
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx))
DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName, filesize);
return 0;
}
@ -2730,8 +2747,10 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
}
if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0)
DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
if (FIO_shouldDisplayMultipleFileSummary(fCtx)) {
DISPLAY_PROGRESS("\r%79s\r", "");
DISPLAY_SUMMARY("%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
}
FIO_freeDResources(ress);
return error;

View File

@ -48,6 +48,14 @@ extern UTIL_time_t g_displayClock;
if (g_display_prefs.displayLevel>=4) fflush(stderr); \
} } }
#define SHOULD_DISPLAY_SUMMARY() \
(g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always)
#define SHOULD_DISPLAY_PROGRESS() \
(g_display_prefs.progressSetting != FIO_ps_never && SHOULD_DISPLAY_SUMMARY())
#define DISPLAY_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYLEVEL(1, __VA_ARGS__); }}
#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }
#undef MIN /* in case it would be already defined */
#define MIN(a,b) ((a) < (b) ? (a) : (b))

View File

@ -0,0 +1,46 @@
#!/bin/sh
#!/bin/sh
. "$COMMON/platform.sh"
set -e
echo hello > hello
echo world > world
zstd -q hello world
println >&2 "Tests cases where progress information should not be printed"
for args in \
"" \
"--fake-stderr-is-console -q" \
"--fake-stderr-is-console -qq --progress" \
"--no-progress --fake-stderr-is-console" \
"--no-progress --fake-stderr-is-console -v"
do
println >&2 "args = $args"
println >&2 "compress file to file"
zstd $args -f hello
println >&2 "compress pipe to pipe"
zstd $args < hello > $INTOVOID
println >&2 "compress pipe to file"
zstd $args < hello -fo hello.zst
println >&2 "compress file to pipe"
zstd $args hello -c > $INTOVOID
println >&2 "compress 2 files"
zstd $args -f hello world
println >&2 "decompress file to file"
zstd $args -d -f hello.zst
println >&2 "decompress pipe to pipe"
zstd $args -d < hello.zst > $INTOVOID
println >&2 "decompress pipe to file"
zstd $args -d < hello.zst -fo hello
println >&2 "decompress file to pipe"
zstd $args -d hello.zst -c > $INTOVOID
println >&2 "decompress 2 files"
zstd $args -d -f hello.zst world.zst
println >&2 ""
done

View File

@ -0,0 +1,90 @@
Tests cases where progress information should not be printed
args =
compress file to file
compress pipe to pipe
compress pipe to file
compress file to pipe
compress 2 files
decompress file to file
decompress pipe to pipe
decompress pipe to file
decompress file to pipe
decompress 2 files
args = --fake-stderr-is-console -q
compress file to file
compress pipe to pipe
compress pipe to file
compress file to pipe
compress 2 files
decompress file to file
decompress pipe to pipe
decompress pipe to file
decompress file to pipe
decompress 2 files
args = --fake-stderr-is-console -qq --progress
compress file to file
compress pipe to pipe
compress pipe to file
compress file to pipe
compress 2 files
decompress file to file
decompress pipe to pipe
decompress pipe to file
decompress file to pipe
decompress 2 files
args = --no-progress --fake-stderr-is-console
compress file to file
hello*hello.zst*
compress pipe to pipe
compress pipe to file
*stdin*hello.zst*
compress file to pipe
compress 2 files
2 files compressed*
decompress file to file
hello.zst*
decompress pipe to pipe
decompress pipe to file
*stdin*
decompress file to pipe
decompress 2 files
2 files decompressed*
args = --no-progress --fake-stderr-is-console -v
compress file to file
*zstd*
hello*hello.zst*
compress pipe to pipe
*zstd*
*stdin*stdout*
compress pipe to file
*zstd*
*stdin*hello.zst*
compress file to pipe
*zstd*
*hello*stdout*
compress 2 files
*zstd*
*hello*hello.zst*
*world*world.zst*
2 files compressed*
decompress file to file
*zstd*
hello.zst*
decompress pipe to pipe
*zstd*
*stdin*
decompress pipe to file
*zstd*
*stdin*
decompress file to pipe
*zstd*
hello.zst*
decompress 2 files
*zstd*
hello.zst*
world.zst*
2 files decompressed*

View File

@ -0,0 +1,41 @@
#!/bin/sh
. "$COMMON/platform.sh"
set -e
println >&2 "Tests cases where progress information should be printed"
echo hello > hello
echo world > world
zstd -q hello world
for args in \
"--progress" \
"--fake-stderr-is-console" \
"--progress --fake-stderr-is-console -q"; do
println >&2 "args = $args"
println >&2 "compress file to file"
zstd $args -f hello
println >&2 "compress pipe to pipe"
zstd $args < hello > $INTOVOID
println >&2 "compress pipe to file"
zstd $args < hello -fo hello.zst
println >&2 "compress file to pipe"
zstd $args hello -c > $INTOVOID
println >&2 "compress 2 files"
zstd $args -f hello world
println >&2 "decompress file to file"
zstd $args -d -f hello.zst
println >&2 "decompress pipe to pipe"
zstd $args -d < hello.zst > $INTOVOID
println >&2 "decompress pipe to file"
zstd $args -d < hello.zst -fo hello
println >&2 "decompress file to pipe"
zstd $args -d hello.zst -c > $INTOVOID
println >&2 "decompress 2 files"
zstd $args -d -f hello.zst world.zst
println >&2 ""
done

View File

@ -0,0 +1,62 @@
Tests cases where progress information should be printed
args = --progress
compress file to file
*Read:*hello*hello.zst*
compress pipe to pipe
*Read:*stdin*stdout*
compress pipe to file
*Read:*stdin*hello.zst*
compress file to pipe
*Read:*hello*stdout*
compress 2 files
*Read*2 files compressed*
decompress file to file
*hello.zst*hello.zst*
decompress pipe to pipe
*stdin*stdin*
decompress pipe to file
*stdin*stdin*
decompress file to pipe
*hello.zst*hello.zst*
decompress 2 files
*hello.zst*2 files decompressed*
args = --fake-stderr-is-console
compress file to file
*Read:*hello*hello.zst*
compress pipe to pipe
compress pipe to file
*Read:*stdin*hello.zst*
compress file to pipe
compress 2 files
*Read*2 files compressed*
decompress file to file
*hello.zst*hello.zst*
decompress pipe to pipe
decompress pipe to file
*stdin*stdin*
decompress file to pipe
decompress 2 files
*hello.zst*2 files decompressed*
args = --progress --fake-stderr-is-console -q
compress file to file
*Read:*hello*hello.zst*
compress pipe to pipe
*Read:*stdin*stdout*
compress pipe to file
*Read:*stdin*hello.zst*
compress file to pipe
*Read:*hello*stdout*
compress 2 files
*Read*2 files compressed*
decompress file to file
*hello.zst*hello.zst*
decompress pipe to pipe
*stdin*stdin*
decompress pipe to file
*stdin*stdin*
decompress file to pipe
*hello.zst*hello.zst*
decompress 2 files
*hello.zst*2 files decompressed*