lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-29 11:21:22 +03:00
Code Activity

Add explicit --pass-through flag and default to enabled for *cat (#3223)

Browse Source 
Fixes #3211.

Adds the `--[no-]pass-through` flag which enables/disables pass-through mode.

* `zstdcat`, `zcat`, and `gzcat` default to `--pass-through`.
  Pass-through mode can be disabled by passing `--no-pass-through`.
* All other binaries default to not setting pass-through mode.
  However, we preserve the legacy behavior of enabling pass-through
  mode when writing to stdout with `-f` set, unless pass-through
  mode is explicitly disabled with `--no-pass-through`.

Adds a new test for this behavior that should codify the behavior we want.
This commit is contained in:
Nick Terrell
2022-08-04 17:15:59 -07:00
committed by GitHub
parent d0dcc9d775
commit 03cc84fddb
7 changed files with 125 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
programs/fileio.c
View File

@ -290,6 +290,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->excludeCompressedFiles = 0;
ret->allowBlockDevices = 0;
ret->asyncIO = AIO_supported();
ret->passThrough = -1;
return ret;
}
@ -463,6 +464,10 @@ void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
#endif
}
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
prefs->passThrough = (value != 0);
}
/* FIO_ctx_t functions */
void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
@ -2336,6 +2341,16 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
{
unsigned readSomething = 0;
unsigned long long filesize = 0;
int passThrough = prefs->passThrough;
if (passThrough == -1) {
/* If pass-through mode is not explicitly enabled or disabled,
* default to the legacy behavior of enabling it if we are writing
* to stdout with the overwrite flag enabled.
*/
passThrough = prefs->overwrite && !strcmp(dstFileName, stdoutmark);
}
assert(passThrough == 0 || passThrough == 1);
/* for each frame */
for ( ; ; ) {
@ -2353,7 +2368,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
}
readSomething = 1; /* there is at least 1 byte in srcFile */
if (ress.readCtx->srcBufferLoaded < toRead) { /* not enough input to check magic number */
if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
if (passThrough) {
return FIO_passThrough(&ress);
}
DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName);
@ -2391,7 +2406,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
return 1;
#endif
} else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
} else if (passThrough) {
return FIO_passThrough(&ress);
} else {
DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);

1
programs/fileio.h
View File

@ -105,6 +105,7 @@ void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
/* FIO_ctx_t functions */
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);

1
programs/fileio_types.h
View File

@ -68,6 +68,7 @@ typedef struct FIO_prefs_s {
int patchFromMode;
int contentSize;
int allowBlockDevices;
int passThrough;
} FIO_prefs_t;
#endif /* FILEIO_TYPES_HEADER */

17
programs/zstdcli.c
View File

@ -264,6 +264,15 @@ static void usage_advanced(const char* programName)
# else
DISPLAYOUT(" --[no-]sparse sparse mode (default: disabled)\n");
# endif
{
char const* passThroughDefault = "disabled";
if (exeNameMatch(programName, ZSTD_CAT) ||
exeNameMatch(programName, ZSTD_ZCAT) ||
exeNameMatch(programName, ZSTD_GZCAT)) {
passThroughDefault = "enabled";
}
DISPLAYOUT(" --[no-]pass-through : passes through uncompressed files as-is (default: %s\n)", passThroughDefault);
}
#endif /* ZSTD_NODECOMPRESS */
#ifndef ZSTD_NODICT
@ -870,14 +879,14 @@ int main(int argCount, const char* argv[])
/* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1);
dictCLevel = cLevel = 6; /* gzip default is -6 */
}
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); } /* behave like gunzip, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like lzma */
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like xz */
@ -926,6 +935,8 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
if (!strcmp(argument, "--pass-through")) { FIO_setPassThroughFlag(prefs, 1); continue; }
if (!strcmp(argument, "--no-pass-through")) { FIO_setPassThroughFlag(prefs, 0); continue; }
if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
if (!strcmp(argument, "--asyncio")) { FIO_setAsyncIOFlag(prefs, 1); continue;}
if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;}