lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

Merge pull request #3232 from facebook/fileiotypes_nomemh

Browse Source 
fileio_types.h : avoid dependency on mem.h
This commit is contained in:
Yann Collet
2022-08-03 22:57:16 +02:00
committed by GitHub
parent 9e90b180c5 3dfcafacd7
commit e818fa8eb0
4 changed files with 23 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
programs/fileio.c
View File

@ -338,7 +338,7 @@ void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t comp
void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }
void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; } void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse) { prefs->sparseFileSupport = sparse; }
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; } void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }
@ -371,7 +371,7 @@ void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
prefs->overlapLog = overlapLog; prefs->overlapLog = overlapLog;
} }
void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) { void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt) {
if ((adapt>0) && (prefs->nbWorkers==0)) if ((adapt>0) && (prefs->nbWorkers==0))
EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n"); EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
prefs->adaptiveMode = adapt; prefs->adaptiveMode = adapt;
@ -453,7 +453,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value)
prefs->contentSize = value != 0; prefs->contentSize = value != 0;
} }
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value) { void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
prefs->asyncIO = value; prefs->asyncIO = value;
#else #else
@ -1306,7 +1306,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
windowLog = ZSTD_WINDOWLOG_LIMIT_DEFAULT; windowLog = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
} else { } else {
const ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, fileSize, 0); const ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, fileSize, 0);
windowLog = cParams.windowLog; windowLog = (int)cParams.windowLog;
} }
} }
windowSize = UTIL_makeHumanReadableSize(MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize))); windowSize = UTIL_makeHumanReadableSize(MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize)));
@ -1726,14 +1726,15 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
return result; return result;
} }
static const char* checked_index(const char* options[], size_t length, size_t index) { static const char*
checked_index(const char* options[], size_t length, size_t index) {
assert(index < length); assert(index < length);
/* Necessary to avoid warnings since -O3 will omit the above `assert` */ /* Necessary to avoid warnings since -O3 will omit the above `assert` */
(void) length; (void) length;
return options[index]; return options[index];
} }
#define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (index)) #define INDEX(options, index) checked_index((options), sizeof(options) / sizeof(char*), (size_t)(index))
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) { void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) {
static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION, static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION,

6
programs/fileio.h
View File

@ -71,7 +71,7 @@ void FIO_freeContext(FIO_ctx_t* const fCtx);
/* FIO_prefs_t functions */ /* FIO_prefs_t functions */
void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType); void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType);
void FIO_overwriteMode(FIO_prefs_t* const prefs); void FIO_overwriteMode(FIO_prefs_t* const prefs);
void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt); void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt);
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel); void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel); void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder); void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
@ -87,7 +87,7 @@ void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit);
void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers); void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers);
void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog); void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag); void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable); void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize); void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize); void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
@ -104,7 +104,7 @@ void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices);
void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
void FIO_setContentSize(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs); void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value); void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
/* FIO_ctx_t functions */ /* FIO_ctx_t functions */
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);

12
programs/fileio_types.h
View File

@ -34,13 +34,13 @@ typedef struct FIO_prefs_s {
/* Algorithm preferences */ /* Algorithm preferences */
FIO_compressionType_t compressionType; FIO_compressionType_t compressionType;
U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
int dictIDFlag; int dictIDFlag;
int checksumFlag; int checksumFlag;
int blockSize; int blockSize;
int overlapLog; int overlapLog;
U32 adaptiveMode; int adaptiveMode;
U32 useRowMatchFinder; int useRowMatchFinder;
int rsyncable; int rsyncable;
int minAdaptLevel; int minAdaptLevel;
int maxAdaptLevel; int maxAdaptLevel;
@ -56,9 +56,9 @@ typedef struct FIO_prefs_s {
ZSTD_paramSwitch_e literalCompressionMode; ZSTD_paramSwitch_e literalCompressionMode;
/* IO preferences */ /* IO preferences */
U32 removeSrcFile; int removeSrcFile;
U32 overwrite; int overwrite;
U32 asyncIO; int asyncIO;
/* Computation resources preferences */ /* Computation resources preferences */
unsigned memLimit; unsigned memLimit;

14
programs/zstdcli.c
View File

@ -801,7 +801,6 @@ int main(int argCount, const char* argv[])
hasStdout = 0, hasStdout = 0,
ldmFlag = 0, ldmFlag = 0,
main_pause = 0, main_pause = 0,
nbWorkers = 0,
adapt = 0, adapt = 0,
useRowMatchFinder = 0, useRowMatchFinder = 0,
adaptMin = MINCLEVEL, adaptMin = MINCLEVEL,
@ -816,6 +815,7 @@ int main(int argCount, const char* argv[])
showDefaultCParams = 0, showDefaultCParams = 0,
ultra=0, ultra=0,
contentSize=1; contentSize=1;
unsigned nbWorkers = 0;
double compressibility = 0.5; double compressibility = 0.5;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0; size_t blockSize = 0;
@ -1200,7 +1200,7 @@ int main(int argCount, const char* argv[])
/* nb of threads (hidden option) */ /* nb of threads (hidden option) */
case 'T': case 'T':
argument++; argument++;
nbWorkers = (int)readU32FromChar(&argument); nbWorkers = readU32FromChar(&argument);
break; break;
/* Dictionary Selection level */ /* Dictionary Selection level */
@ -1246,10 +1246,10 @@ int main(int argCount, const char* argv[])
if ((nbWorkers==0) && (!singleThread)) { if ((nbWorkers==0) && (!singleThread)) {
/* automatically set # workers based on # of reported cpus */ /* automatically set # workers based on # of reported cpus */
if (defaultLogicalCores) { if (defaultLogicalCores) {
nbWorkers = UTIL_countLogicalCores(); nbWorkers = (unsigned)UTIL_countLogicalCores();
DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers); DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
} else { } else {
nbWorkers = UTIL_countPhysicalCores(); nbWorkers = (unsigned)UTIL_countPhysicalCores();
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers); DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
} }
} }
@ -1313,7 +1313,7 @@ int main(int argCount, const char* argv[])
if (operation==zom_bench) { if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
benchParams.blockSize = blockSize; benchParams.blockSize = blockSize;
benchParams.nbWorkers = nbWorkers; benchParams.nbWorkers = (int)nbWorkers;
benchParams.realTime = (unsigned)setRealTimePrio; benchParams.realTime = (unsigned)setRealTimePrio;
benchParams.nbSeconds = bench_nbSeconds; benchParams.nbSeconds = bench_nbSeconds;
benchParams.ldmFlag = ldmFlag; benchParams.ldmFlag = ldmFlag;
@ -1474,7 +1474,7 @@ int main(int argCount, const char* argv[])
if (operation==zom_compress) { if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
FIO_setContentSize(prefs, contentSize); FIO_setContentSize(prefs, contentSize);
FIO_setNbWorkers(prefs, nbWorkers); FIO_setNbWorkers(prefs, (int)nbWorkers);
FIO_setBlockSize(prefs, (int)blockSize); FIO_setBlockSize(prefs, (int)blockSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog); if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
FIO_setLdmFlag(prefs, (unsigned)ldmFlag); FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
@ -1482,7 +1482,7 @@ int main(int argCount, const char* argv[])
FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch); FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog); if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog); if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
FIO_setAdaptiveMode(prefs, (unsigned)adapt); FIO_setAdaptiveMode(prefs, adapt);
FIO_setUseRowMatchFinder(prefs, useRowMatchFinder); FIO_setUseRowMatchFinder(prefs, useRowMatchFinder);
FIO_setAdaptMin(prefs, adaptMin); FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(prefs, adaptMax); FIO_setAdaptMax(prefs, adaptMax);