From da38891a87dc1be9796a65ce2c8214d16fe49379 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 25 Aug 2020 16:46:47 -0400 Subject: [PATCH 1/9] Add initial live update with displayLevel = 2, add new field to FIO_prefs_t --- programs/fileio.c | 48 ++++++++++++++++++++++++++++++---------------- programs/fileio.h | 1 + programs/zstdcli.c | 15 +++++++-------- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index d5b8a7d14..079b3bc43 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -319,6 +319,7 @@ struct FIO_prefs_s { int excludeCompressedFiles; int patchFromMode; int contentSize; + int nbFiles; }; @@ -360,6 +361,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->testMode = 0; ret->literalCompressionMode = ZSTD_lcm_auto; ret->excludeCompressedFiles = 0; + ret->nbFiles = 1; return ret; } @@ -495,6 +497,11 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value) prefs->contentSize = value != 0; } +void FIO_setNbFiles(FIO_prefs_t* const prefs, int value) +{ + prefs->nbFiles = value; +} + /*-************************************* * Functions ***************************************/ @@ -1254,17 +1261,17 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, /* display progress notifications */ if (g_display_prefs.displayLevel >= 3) { - DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", + DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%%\033 ", compressionLevel, (unsigned)((zfp.ingested - zfp.consumed) >> 20), (unsigned)(zfp.consumed >> 20), (unsigned)(zfp.produced >> 20), cShare ); } else { /* summarized notifications if == 2; */ - DISPLAYLEVEL(2, "\rRead : %u ", (unsigned)(zfp.consumed >> 20)); + DISPLAYLEVEL(2, "\033[s Read : %u ", (unsigned)(zfp.consumed >> 20)); if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20)); - DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare); + DISPLAYLEVEL(2, "MB ==> %2.f%%\033[u", cShare); DELAY_NEXT_UPDATE(); } @@ -1427,18 +1434,21 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, } /* Status */ - DISPLAYLEVEL(2, "\r%79s\r", ""); - if (readsize == 0) { - DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", - srcFileName, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); - } else { - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", - srcFileName, - (double)compressedfilesize / readsize * 100, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); + + if (prefs->nbFiles == 1 && !((!strcmp(srcFileName, stdinmark) && dstFileName && !strcmp(dstFileName,stdoutmark)))) { + DISPLAYLEVEL(2, "\r%79s\r", ""); + if (readsize == 0) { + DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", + srcFileName, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); + } else { + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", + srcFileName, + (double)compressedfilesize / readsize * 100, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); + } } /* Elapsed Time and CPU Load */ @@ -1682,8 +1692,10 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, error = 1; } else { unsigned u; - for (u=0; u 1) + DISPLAYUPDATE(2, "\rCompressing %u/%u files. Current source: %s |", u+1, nbFiles, srcFileName); error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); } diff --git a/programs/fileio.h b/programs/fileio.h index ef3c9e72f..5913c8a13 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -96,6 +96,7 @@ void FIO_setNotificationLevel(int level); void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles); void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); +void FIO_setNbFiles(FIO_prefs_t* const prefs, int value); /*-************************************* * Single File functions diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 1aea63dca..534a848e2 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1244,10 +1244,6 @@ int main(int const argCount, const char* argv[]) CLEAN_RETURN(1); } - /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; - if ((filenames->tableSize > 1) & (g_displayLevel==2)) g_displayLevel=1; - /* IO Stream/File */ FIO_setNotificationLevel(g_displayLevel); FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL); @@ -1306,10 +1302,12 @@ int main(int const argCount, const char* argv[]) } } - if ((filenames->tableSize==1) && outFileName) - operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); - else - operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); + if ((filenames->tableSize==1) && outFileName) { + operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); + } else { + FIO_setNbFiles(prefs, (int)filenames->tableSize); + operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); + } #else (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); @@ -1319,6 +1317,7 @@ int main(int const argCount, const char* argv[]) if (filenames->tableSize == 1 && outFileName) { operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName); } else { + FIO_setNbFiles(prefs, (int)filenames->tableSize); operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outMirroredDirName, outDirName, outFileName, dictFileName); } #else From a3401ca7d0deae37f21bd2d6fa1262320b5675da Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 25 Aug 2020 17:23:47 -0400 Subject: [PATCH 2/9] Make compatible with displaylevel = 2 and decompression --- programs/fileio.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 079b3bc43..c81b640e1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1435,8 +1435,9 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, /* Status */ + DISPLAYLEVEL(2, "\r%79s\r", ""); + /* No status message in pipe mode (stdin - stdout) or multi-files mode */ if (prefs->nbFiles == 1 && !((!strcmp(srcFileName, stdinmark) && dstFileName && !strcmp(dstFileName,stdoutmark)))) { - DISPLAYLEVEL(2, "\r%79s\r", ""); if (readsize == 0) { DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", srcFileName, @@ -1693,7 +1694,8 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } else { unsigned u; for (u=0; u 1) + DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s | ", u+1, nbFiles, inFileNamesTable[u]); error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[u], compressionLevel); } if (fclose(ress.dstFile)) @@ -1726,7 +1728,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, /* No status message in pipe mode (stdin - stdout) or multi-files mode */ // if (!strcmp(inFileNamesTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_display_prefs.displayLevel==2)) g_displayLevel=1; if (nbFiles > 1) - DISPLAYUPDATE(2, "\rCompressing %u/%u files. Current source: %s |", u+1, nbFiles, srcFileName); + DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s | ", u+1, nbFiles, srcFileName); error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); } @@ -2001,7 +2003,7 @@ FIO_decompressZstdFrame(dRess_t* ress, FILE* finput, /* Write block */ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); frameSize += outBuff.pos; - DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", + DISPLAYUPDATE(2, "\033[s%-20.20s : %u MB... \033[u", srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); if (inBuff.pos > 0) { @@ -2584,8 +2586,10 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); } - for (u=0; utestMode) && (fclose(ress.dstFile))) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); @@ -2609,6 +2613,8 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, dstFileName = FIO_determineDstName(srcFileName, outDirName); } if (dstFileName == NULL) { error=1; continue; } + if (nbFiles > 1) + DISPLAYLEVEL(2, "\rDecompressing %u/%u files. Current source: %s | ", u+1, nbFiles, srcFileName[u]); error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); } if (outDirName) From acee48d4c1e73a0a8a67f7c9fdb8be15e939a197 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 25 Aug 2020 17:34:00 -0400 Subject: [PATCH 3/9] Small cleanups --- programs/fileio.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c81b640e1..25d980980 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1261,7 +1261,7 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, /* display progress notifications */ if (g_display_prefs.displayLevel >= 3) { - DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%%\033 ", + DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", compressionLevel, (unsigned)((zfp.ingested - zfp.consumed) >> 20), (unsigned)(zfp.consumed >> 20), @@ -1725,8 +1725,6 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ } - /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - // if (!strcmp(inFileNamesTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_display_prefs.displayLevel==2)) g_displayLevel=1; if (nbFiles > 1) DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s | ", u+1, nbFiles, srcFileName); error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -2614,7 +2612,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } if (dstFileName == NULL) { error=1; continue; } if (nbFiles > 1) - DISPLAYLEVEL(2, "\rDecompressing %u/%u files. Current source: %s | ", u+1, nbFiles, srcFileName[u]); + DISPLAYLEVEL(2, "\rDecompressing %u/%u files. Current source: %s | ", u+1, nbFiles, srcFileName); error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); } if (outDirName) From 47a9ad2ae3b21680f24ff8b78b8834eeee46ffc1 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 25 Aug 2020 17:54:00 -0400 Subject: [PATCH 4/9] Remove decompression status update --- programs/fileio.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 25d980980..4be316963 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1437,7 +1437,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, DISPLAYLEVEL(2, "\r%79s\r", ""); /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if (prefs->nbFiles == 1 && !((!strcmp(srcFileName, stdinmark) && dstFileName && !strcmp(dstFileName,stdoutmark)))) { + if (g_display_prefs.displayLevel > 2 || (prefs->nbFiles == 1 && !((!strcmp(srcFileName, stdinmark) && dstFileName && !strcmp(dstFileName,stdoutmark))))) { if (readsize == 0) { DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", srcFileName, @@ -2332,7 +2332,9 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); + /* No status message in pipe mode (stdin - stdout) or multi-files mode */ + if (g_display_prefs.displayLevel > 2 || (prefs->nbFiles == 1 && !((!strcmp(srcFileName, stdinmark) && dstFileName && !strcmp(dstFileName,stdoutmark))))) + DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); return 0; } @@ -2585,7 +2587,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); } for (u=0; utestMode) && (fclose(ress.dstFile))) @@ -2612,7 +2614,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } if (dstFileName == NULL) { error=1; continue; } if (nbFiles > 1) - DISPLAYLEVEL(2, "\rDecompressing %u/%u files. Current source: %s | ", u+1, nbFiles, srcFileName); + DISPLAYLEVEL(2, "\rDecompressing %u/%u files. Current source: ", u+1, nbFiles); error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); } if (outDirName) From 96222cd195193715204292ea26a545582554f4b6 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 25 Aug 2020 18:03:08 -0400 Subject: [PATCH 5/9] Small formatting changes --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 4be316963..20166b309 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1695,7 +1695,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, unsigned u; for (u=0; u 1) - DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s | ", u+1, nbFiles, inFileNamesTable[u]); + DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", u+1, nbFiles, inFileNamesTable[u]); error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[u], compressionLevel); } if (fclose(ress.dstFile)) @@ -1726,7 +1726,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } if (nbFiles > 1) - DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s | ", u+1, nbFiles, srcFileName); + DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", u+1, nbFiles, srcFileName); error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); } From b6abbc3ce510c9b62d1d762d5fc010b4eca58e8e Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 26 Aug 2020 11:35:07 -0400 Subject: [PATCH 6/9] Fix single file decompression summary, remove escape code dependency, add currFileIdx field to FIO_prefs, general cleanups/bugfixes --- programs/fileio.c | 75 ++++++++++++++++++++++++---------------------- programs/fileio.h | 5 ++-- programs/zstdcli.c | 14 +++++---- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 20166b309..29b499b4b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -319,6 +319,7 @@ struct FIO_prefs_s { int excludeCompressedFiles; int patchFromMode; int contentSize; + int currFileIdx; int nbFiles; }; @@ -362,6 +363,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->literalCompressionMode = ZSTD_lcm_auto; ret->excludeCompressedFiles = 0; ret->nbFiles = 1; + ret->currFileIdx = 0; return ret; } @@ -502,6 +504,11 @@ void FIO_setNbFiles(FIO_prefs_t* const prefs, int value) prefs->nbFiles = value; } +void FIO_setCurrFileIdx(FIO_prefs_t* const prefs, int value) +{ + prefs->currFileIdx = value; +} + /*-************************************* * Functions ***************************************/ @@ -1268,10 +1275,15 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, (unsigned)(zfp.produced >> 20), cShare ); } else { /* summarized notifications if == 2; */ - DISPLAYLEVEL(2, "\033[s Read : %u ", (unsigned)(zfp.consumed >> 20)); + if (prefs->nbFiles > 1) { + DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", prefs->currFileIdx+1, prefs->nbFiles, srcFileName); + } else { + DISPLAYLEVEL(2, "\r"); + } + DISPLAYLEVEL(2, "Read : %u ", (unsigned)(zfp.consumed >> 20)); if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20)); - DISPLAYLEVEL(2, "MB ==> %2.f%%\033[u", cShare); + DISPLAYLEVEL(2, "MB ==> %2.f%%", cShare); DELAY_NEXT_UPDATE(); } @@ -1436,8 +1448,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if (g_display_prefs.displayLevel > 2 || (prefs->nbFiles == 1 && !((!strcmp(srcFileName, stdinmark) && dstFileName && !strcmp(dstFileName,stdoutmark))))) { + if (g_display_prefs.displayLevel > 2 || (g_display_prefs.displayLevel == 2 && prefs->nbFiles <= 1)) { if (readsize == 0) { DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", srcFileName, @@ -1673,7 +1684,7 @@ static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsig * or into a destination folder (specified with -O) */ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** inFileNamesTable, unsigned nbFiles, + const char** inFileNamesTable, const char* outMirroredRootDirName, const char* outDirName, const char* outFileName, const char* suffix, @@ -1682,7 +1693,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, { int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, - FIO_getLargestFileSize(inFileNamesTable, nbFiles), + FIO_getLargestFileSize(inFileNamesTable, prefs->nbFiles), compressionLevel, comprParams); /* init */ @@ -1692,11 +1703,8 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; } else { - unsigned u; - for (u=0; u 1) - DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", u+1, nbFiles, inFileNamesTable[u]); - error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[u], compressionLevel); + for (; prefs->currFileIdx < prefs->nbFiles; ++prefs->currFileIdx) { + error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[prefs->currFileIdx], compressionLevel); } if (fclose(ress.dstFile)) EXM_THROW(29, "Write error (%s) : cannot properly close %s", @@ -1704,12 +1712,11 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, ress.dstFile = NULL; } } else { - unsigned int u=0; if (outMirroredRootDirName) - UTIL_mirrorSourceFilesDirectories(inFileNamesTable, nbFiles, outMirroredRootDirName); + UTIL_mirrorSourceFilesDirectories(inFileNamesTable, prefs->nbFiles, outMirroredRootDirName); - for (u=0; ucurrFileIdx < prefs->nbFiles; ++prefs->currFileIdx) { + const char* const srcFileName = inFileNamesTable[prefs->currFileIdx]; const char* dstFileName = NULL; if (outMirroredRootDirName) { char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); @@ -1724,14 +1731,11 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } else { dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ } - - if (nbFiles > 1) - DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", u+1, nbFiles, srcFileName); error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); } if (outDirName) - FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles); + FIO_checkFilenameCollisions(inFileNamesTable , prefs->nbFiles); } FIO_freeCResources(ress); @@ -2001,8 +2005,13 @@ FIO_decompressZstdFrame(dRess_t* ress, FILE* finput, /* Write block */ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); frameSize += outBuff.pos; - DISPLAYUPDATE(2, "\033[s%-20.20s : %u MB... \033[u", - srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + if (prefs->nbFiles > 1) { + DISPLAYUPDATE(2, "\rDecompressing %u/%u files. Current source: %-20.20s : %u MB... ", + prefs->currFileIdx+1, prefs->nbFiles, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + } else { + DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", + srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + } if (inBuff.pos > 0) { memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); @@ -2333,8 +2342,9 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if (g_display_prefs.displayLevel > 2 || (prefs->nbFiles == 1 && !((!strcmp(srcFileName, stdinmark) && dstFileName && !strcmp(dstFileName,stdoutmark))))) - DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); + if (g_display_prefs.displayLevel > 2 || (g_display_prefs.displayLevel == 2 && prefs->nbFiles <= 1)) { + DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); + } return 0; } @@ -2572,7 +2582,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, - const char** srcNamesTable, unsigned nbFiles, + const char** srcNamesTable, const char* outMirroredRootDirName, const char* outDirName, const char* outFileName, const char* dictFileName) @@ -2581,25 +2591,22 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, dRess_t ress = FIO_createDResources(prefs, dictFileName); if (outFileName) { - unsigned u; if (!prefs->testMode) { ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); } - for (u=0; ucurrFileIdx < prefs->nbFiles; prefs->currFileIdx++) { + error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[prefs->currFileIdx]); } if ((!prefs->testMode) && (fclose(ress.dstFile))) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); } else { - unsigned int u = 0; if (outMirroredRootDirName) - UTIL_mirrorSourceFilesDirectories(srcNamesTable, nbFiles, outMirroredRootDirName); + UTIL_mirrorSourceFilesDirectories(srcNamesTable, prefs->nbFiles, outMirroredRootDirName); - for (u=0; ucurrFileIdx < prefs->nbFiles; prefs->currFileIdx++) { /* create dstFileName */ + const char* const srcFileName = srcNamesTable[prefs->currFileIdx]; const char* dstFileName = NULL; if (outMirroredRootDirName) { char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); @@ -2613,12 +2620,10 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, dstFileName = FIO_determineDstName(srcFileName, outDirName); } if (dstFileName == NULL) { error=1; continue; } - if (nbFiles > 1) - DISPLAYLEVEL(2, "\rDecompressing %u/%u files. Current source: ", u+1, nbFiles); error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); } if (outDirName) - FIO_checkFilenameCollisions(srcNamesTable ,nbFiles); + FIO_checkFilenameCollisions(srcNamesTable , prefs->nbFiles); } FIO_freeDResources(ress); diff --git a/programs/fileio.h b/programs/fileio.h index 5913c8a13..f86ae4178 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -97,6 +97,7 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); void FIO_setNbFiles(FIO_prefs_t* const prefs, int value); +void FIO_setCurrFileIdx(FIO_prefs_t* const prefs, int value); /*-************************************* * Single File functions @@ -122,7 +123,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis /** FIO_compressMultipleFilenames() : * @return : nb of missing files */ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** inFileNamesTable, unsigned nbFiles, + const char** inFileNamesTable, const char* outMirroredDirName, const char* outDirName, const char* outFileName, const char* suffix, @@ -132,7 +133,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, /** FIO_decompressMultipleFilenames() : * @return : nb of missing or skipped files */ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, - const char** srcNamesTable, unsigned nbFiles, + const char** srcNamesTable, const char* outMirroredDirName, const char* outDirName, const char* outFileName, diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 534a848e2..9d4eee5d8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1243,8 +1243,12 @@ int main(int const argCount, const char* argv[]) DISPLAY("error : can't use --patch-from=# on multiple files \n"); CLEAN_RETURN(1); } + + /* No status message in pipe mode (stdin - stdout) */ + if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; /* IO Stream/File */ + FIO_setNbFiles(prefs, (int)filenames->tableSize); FIO_setNotificationLevel(g_displayLevel); FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL); if (memLimit == 0) { @@ -1302,12 +1306,10 @@ int main(int const argCount, const char* argv[]) } } - if ((filenames->tableSize==1) && outFileName) { + if ((filenames->tableSize==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); - } else { - FIO_setNbFiles(prefs, (int)filenames->tableSize); - operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); - } + else + operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); @@ -1318,7 +1320,7 @@ int main(int const argCount, const char* argv[]) operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName); } else { FIO_setNbFiles(prefs, (int)filenames->tableSize); - operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outMirroredDirName, outDirName, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName); } #else DISPLAY("Decompression not supported \n"); From a73e131f100bce734f3d9472a6f1606699f95574 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 26 Aug 2020 11:40:05 -0400 Subject: [PATCH 7/9] Adjust playTests.sh refuse overwrite test to include -q --- tests/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index b7bfa76cd..73b932808 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -361,7 +361,7 @@ zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f zstd -d tmp1.zst tmp2.zst -o tmp touch tmpexists zstd tmp1 tmp2 -f -o tmpexists -zstd tmp1 tmp2 -o tmpexists && die "should have refused to overwrite" +zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite" # Bug: PR #972 if [ "$?" -eq 139 ]; then die "should not have segfaulted" From fed7e7850bcc3267f5fbda167c0281d48b91758f Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 26 Aug 2020 11:57:38 -0400 Subject: [PATCH 8/9] Fix bug in user prompt where line was not flushed on negative response --- programs/fileio.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 29b499b4b..e4c958487 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -624,9 +624,10 @@ FIO_openDstFile(FIO_prefs_t* const prefs, { int ch = getchar(); if ((ch!='Y') && (ch!='y')) { DISPLAY(" not overwritten \n"); + /* flush rest of input line */ + while ((ch!=EOF) && (ch!='\n')) ch = getchar(); return NULL; } - /* flush rest of input line */ while ((ch!=EOF) && (ch!='\n')) ch = getchar(); } } /* need to unlink */ From ead2387a7e77f6b3ee38a098556fb9143c7db5e4 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 26 Aug 2020 12:05:04 -0400 Subject: [PATCH 9/9] Remove extraneous FIO_setNbFiles() --- programs/zstdcli.c | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9d4eee5d8..79113ee41 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1319,7 +1319,6 @@ int main(int const argCount, const char* argv[]) if (filenames->tableSize == 1 && outFileName) { operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName); } else { - FIO_setNbFiles(prefs, (int)filenames->tableSize); operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName); } #else