mirror of
https://github.com/facebook/zstd.git
synced 2025-08-01 09:47:01 +03:00
Merge branch 'master' of https://github.com/bimbashrestha/zstd into named_pipes
This commit is contained in:
Bimba Shrestha
@ -139,7 +139,7 @@ static int usage_advanced(const char* programName)
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
|
||||
DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
|
||||
DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
|
||||
DISPLAY( "--fast[=#]: switch to very fast compression levels (default: %u)\n", 1);
|
||||
DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n");
|
||||
DISPLAY( "--stream-size=# : optimize compression parameters for streaming input of given number of bytes \n");
|
||||
DISPLAY( "--size-hint=# optimize compression parameters for streaming input of approximately this size\n");
|
||||
@ -147,7 +147,7 @@ static int usage_advanced(const char* programName)
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
|
||||
DISPLAY( " -B# : select size of each job (default: 0==automatic) \n");
|
||||
DISPLAY( " --rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
|
||||
DISPLAY( "--rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
|
||||
#endif
|
||||
DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
|
||||
DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
|
||||
@ -155,6 +155,7 @@ static int usage_advanced(const char* programName)
|
||||
#endif
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
DISPLAY( " -r : operate recursively on directories \n");
|
||||
DISPLAY( "--output-dir-flat[=directory]: all resulting files stored into `directory`. \n");
|
||||
#endif
|
||||
DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
|
||||
#ifdef ZSTD_GZCOMPRESS
|
||||
@ -286,7 +287,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
|
||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||
*/
|
||||
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
static int longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||
{
|
||||
size_t const comSize = strlen(longCommand);
|
||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||
@ -429,8 +430,8 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void)
|
||||
static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
|
||||
{
|
||||
for ( ; ;) {
|
||||
if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||
DISPLAYLEVEL(4, "invalid compression parameter \n");
|
||||
return 0;
|
||||
}
|
||||
@ -525,7 +526,7 @@ static int init_cLevel(void) {
|
||||
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
|
||||
return ZSTDCLI_CLEVEL_DEFAULT;
|
||||
} else if (*ptr == 0) {
|
||||
return sign * absLevel;
|
||||
return sign * (int)absLevel;
|
||||
}
|
||||
}
|
||||
|
||||
@ -562,6 +563,7 @@ int main(int argCount, const char* argv[])
|
||||
adaptMax = MAXCLEVEL,
|
||||
rsyncable = 0,
|
||||
nextArgumentIsOutFileName = 0,
|
||||
nextArgumentIsOutDirName = 0,
|
||||
nextArgumentIsMaxDict = 0,
|
||||
nextArgumentIsDictID = 0,
|
||||
nextArgumentsAreFiles = 0,
|
||||
@ -582,10 +584,11 @@ int main(int argCount, const char* argv[])
|
||||
int cLevelLast = -1000000000;
|
||||
unsigned recursive = 0;
|
||||
unsigned memLimit = 0;
|
||||
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
|
||||
const char** filenameTable = (const char**)malloc((size_t)argCount * sizeof(const char*)); /* argCount >= 1 */
|
||||
unsigned filenameIdx = 0;
|
||||
const char* programName = argv[0];
|
||||
const char* outFileName = NULL;
|
||||
const char* outDirName = NULL;
|
||||
const char* dictFileName = NULL;
|
||||
const char* suffix = ZSTD_EXTENSION;
|
||||
unsigned maxDictSize = g_defaultMaxDictSize;
|
||||
@ -686,6 +689,7 @@ int main(int argCount, const char* argv[])
|
||||
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
|
||||
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
|
||||
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
|
||||
if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; }
|
||||
if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
|
||||
if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; }
|
||||
if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
|
||||
@ -741,7 +745,7 @@ int main(int argCount, const char* argv[])
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
|
||||
@ -752,6 +756,7 @@ int main(int argCount, const char* argv[])
|
||||
if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; }
|
||||
if (longCommandWArg(&argument, "--long")) {
|
||||
unsigned ldmWindowLog = 0;
|
||||
ldmFlag = 1;
|
||||
@ -803,7 +808,7 @@ int main(int argCount, const char* argv[])
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
/* compression Level */
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
dictCLevel = cLevel = readU32FromChar(&argument);
|
||||
dictCLevel = cLevel = (int)readU32FromChar(&argument);
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
@ -875,7 +880,7 @@ int main(int argCount, const char* argv[])
|
||||
case 'e':
|
||||
/* compression Level */
|
||||
argument++;
|
||||
cLevelLast = readU32FromChar(&argument);
|
||||
cLevelLast = (int)readU32FromChar(&argument);
|
||||
break;
|
||||
|
||||
/* Modify Nb Iterations (benchmark only) */
|
||||
@ -901,7 +906,7 @@ int main(int argCount, const char* argv[])
|
||||
/* nb of threads (hidden option) */
|
||||
case 'T':
|
||||
argument++;
|
||||
nbWorkers = readU32FromChar(&argument);
|
||||
nbWorkers = (int)readU32FromChar(&argument);
|
||||
break;
|
||||
|
||||
/* Dictionary Selection level */
|
||||
@ -965,6 +970,13 @@ int main(int argCount, const char* argv[])
|
||||
continue;
|
||||
}
|
||||
|
||||
if (nextArgumentIsOutDirName) {
|
||||
nextArgumentIsOutDirName = 0;
|
||||
lastCommand = 0;
|
||||
outDirName = argument;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* add filename to list */
|
||||
filenameTable[filenameIdx++] = argument;
|
||||
}
|
||||
@ -1035,16 +1047,16 @@ int main(int argCount, const char* argv[])
|
||||
#ifndef ZSTD_NOBENCH
|
||||
benchParams.blockSize = blockSize;
|
||||
benchParams.nbWorkers = nbWorkers;
|
||||
benchParams.realTime = setRealTimePrio;
|
||||
benchParams.realTime = (unsigned)setRealTimePrio;
|
||||
benchParams.nbSeconds = bench_nbSeconds;
|
||||
benchParams.ldmFlag = ldmFlag;
|
||||
benchParams.ldmMinMatch = g_ldmMinMatch;
|
||||
benchParams.ldmHashLog = g_ldmHashLog;
|
||||
benchParams.ldmMinMatch = (int)g_ldmMinMatch;
|
||||
benchParams.ldmHashLog = (int)g_ldmHashLog;
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
|
||||
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
|
||||
benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
|
||||
}
|
||||
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
|
||||
benchParams.ldmHashRateLog = g_ldmHashRateLog;
|
||||
benchParams.ldmHashRateLog = (int)g_ldmHashRateLog;
|
||||
}
|
||||
benchParams.literalCompressionMode = literalCompressionMode;
|
||||
|
||||
@ -1085,16 +1097,16 @@ int main(int argCount, const char* argv[])
|
||||
#ifndef ZSTD_NODICT
|
||||
ZDICT_params_t zParams;
|
||||
zParams.compressionLevel = dictCLevel;
|
||||
zParams.notificationLevel = g_displayLevel;
|
||||
zParams.notificationLevel = (unsigned)g_displayLevel;
|
||||
zParams.dictID = dictID;
|
||||
if (dict == cover) {
|
||||
int const optimize = !coverParams.k || !coverParams.d;
|
||||
coverParams.nbThreads = nbWorkers;
|
||||
coverParams.nbThreads = (unsigned)nbWorkers;
|
||||
coverParams.zParams = zParams;
|
||||
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
|
||||
} else if (dict == fastCover) {
|
||||
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
|
||||
fastCoverParams.nbThreads = nbWorkers;
|
||||
fastCoverParams.nbThreads = (unsigned)nbWorkers;
|
||||
fastCoverParams.zParams = zParams;
|
||||
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
|
||||
} else {
|
||||
@ -1113,7 +1125,7 @@ int main(int argCount, const char* argv[])
|
||||
}
|
||||
|
||||
#ifndef ZSTD_NODECOMPRESS
|
||||
if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
|
||||
if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
|
||||
#endif
|
||||
|
||||
/* No input filename ==> use stdin and stdout */
|
||||
@ -1149,14 +1161,14 @@ int main(int argCount, const char* argv[])
|
||||
if (operation==zom_compress) {
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
FIO_setNbWorkers(prefs, nbWorkers);
|
||||
FIO_setBlockSize(prefs, (U32)blockSize);
|
||||
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog);
|
||||
FIO_setLdmFlag(prefs, ldmFlag);
|
||||
FIO_setLdmHashLog(prefs, g_ldmHashLog);
|
||||
FIO_setLdmMinMatch(prefs, g_ldmMinMatch);
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog);
|
||||
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog);
|
||||
FIO_setAdaptiveMode(prefs, adapt);
|
||||
FIO_setBlockSize(prefs, (int)blockSize);
|
||||
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
|
||||
FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
|
||||
FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
|
||||
FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
|
||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
|
||||
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
|
||||
FIO_setAdaptiveMode(prefs, (unsigned)adapt);
|
||||
FIO_setAdaptMin(prefs, adaptMin);
|
||||
FIO_setAdaptMax(prefs, adaptMax);
|
||||
FIO_setRsyncable(prefs, rsyncable);
|
||||
@ -1170,7 +1182,7 @@ int main(int argCount, const char* argv[])
|
||||
if ((filenameIdx==1) && outFileName)
|
||||
operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
|
||||
else
|
||||
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
||||
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
||||
#else
|
||||
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
|
||||
DISPLAY("Compression not supported \n");
|
||||
@ -1188,7 +1200,7 @@ int main(int argCount, const char* argv[])
|
||||
if (filenameIdx==1 && outFileName)
|
||||
operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
|
||||
else
|
||||
operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, dictFileName);
|
||||
operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName);
|
||||
#else
|
||||
DISPLAY("Decompression not supported \n");
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user