lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-07 06:23:00 +03:00
Code Activity

ability to disable mmap + struct to manage FIO dictionary

Browse Source
This commit is contained in:
Danielle Rozenblit
2023-03-08 08:06:10 -08:00
parent 2d8afd9ce1
commit 96e55c14f2
4 changed files with 48 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
programs/fileio.c
View File

@@ -485,7 +485,7 @@ void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
prefs->passThrough = (value != 0); prefs->passThrough = (value != 0);
} }
void FIO_setMMapDict(FIO_prefs_t* const prefs, int value) void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_paramSwitch_e value)
{ {
prefs->mmapDict = value; prefs->mmapDict = value;
} }
@@ -769,20 +769,24 @@ static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, F
close(fileHandle); close(fileHandle);
return (size_t)fileSize; return (size_t)fileSize;
} }
static void FIO_munmapDictBuffer(void* dictBuffer, size_t dictBufferSize) {
FIO_munmap(dictBuffer, dictBufferSize);
}
#else #else
static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat) static size_t FIO_createDictBufferMMap(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{ {
return FIO_createDictBuffer(bufferPtr, fileName, prefs, dictFileStat); return FIO_createDictBuffer(bufferPtr, fileName, prefs, dictFileStat);
} }
static void FIO_munmapDictBuffer(void* dictBuffer, size_t dictBufferSize) { static void FIO_munmap(void* buffer, size_t bufferSize) {
(void)dictBufferSize; (void)bufferSize;
free(dictBuffer); free(buffer);
} }
#endif #endif
static void FIO_freeDict(const FIO_Dict_t* dict) {
if (dict->dictBufferType == FIO_mallocDict) {
free(dict->dictBuffer);
} else if (dict->dictBufferType == FIO_mmapDict) {
FIO_munmap(dict->dictBuffer, dict->dictBufferSize);
}
}
/* FIO_checkFilenameCollisions() : /* FIO_checkFilenameCollisions() :
@@ -985,14 +989,12 @@ static ZSTD_outBuffer setOutBuffer(void* buf, size_t s, size_t pos)
* Compression * Compression
************************************************************************/ ************************************************************************/
typedef struct { typedef struct {
void* dictBuffer; FIO_Dict_t dict;
size_t dictBufferSize;
const char* dictFileName; const char* dictFileName;
stat_t dictFileStat; stat_t dictFileStat;
ZSTD_CStream* cctx; ZSTD_CStream* cctx;
WritePoolCtx_t *writeCtx; WritePoolCtx_t *writeCtx;
ReadPoolCtx_t *readCtx; ReadPoolCtx_t *readCtx;
int mmapDict;
} cRess_t; } cRess_t;
/** ZSTD_cycleLog() : /** ZSTD_cycleLog() :
@@ -1033,7 +1035,8 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
const char* dictFileName, unsigned long long const maxSrcFileSize, const char* dictFileName, unsigned long long const maxSrcFileSize,
int cLevel, ZSTD_compressionParameters comprParams) { int cLevel, ZSTD_compressionParameters comprParams) {
int mmapDict = prefs->mmapDict; int useMMap = prefs->mmapDict == ZSTD_ps_enable;
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
cRess_t ress; cRess_t ress;
memset(&ress, 0, sizeof(ress)); memset(&ress, 0, sizeof(ress));
@@ -1050,23 +1053,23 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
if (prefs->patchFromMode) { if (prefs->patchFromMode) {
U64 const dictSize = UTIL_getFileSizeStat(&ress.dictFileStat); U64 const dictSize = UTIL_getFileSizeStat(&ress.dictFileStat);
unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize; unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
mmapDict |= dictSize > prefs->memLimit; useMMap |= dictSize > prefs->memLimit;
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel); FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
} }
ress.mmapDict = mmapDict; ress.dict.dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
if (!ress.mmapDict) { if (ress.dict.dictBufferType == FIO_mallocDict) {
ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs, &ress.dictFileStat); /* works with dictFileName==NULL */ ress.dict.dictBufferSize = FIO_createDictBuffer(&ress.dict.dictBuffer, dictFileName, prefs, &ress.dictFileStat); /* works with dictFileName==NULL */
} else { } else {
ress.dictBufferSize = FIO_createDictBufferMMap(&ress.dictBuffer, dictFileName, prefs, &ress.dictFileStat); ress.dict.dictBufferSize = FIO_createDictBufferMMap(&ress.dict.dictBuffer, dictFileName, prefs, &ress.dictFileStat);
} }
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize()); ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize()); ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
/* Advanced parameters, including dictionary */ /* Advanced parameters, including dictionary */
if (dictFileName && (ress.dictBuffer==NULL)) if (dictFileName && (ress.dict.dictBuffer==NULL))
EXM_THROW(32, "allocation error : can't create dictBuffer"); EXM_THROW(32, "allocation error : can't create dictBuffer");
ress.dictFileName = dictFileName; ress.dictFileName = dictFileName;
@@ -1116,9 +1119,9 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
#endif #endif
/* dictionary */ /* dictionary */
if (prefs->patchFromMode) { if (prefs->patchFromMode) {
CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dict.dictBuffer, ress.dict.dictBufferSize) );
} else { } else {
CHECK( ZSTD_CCtx_loadDictionary_byReference(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); CHECK( ZSTD_CCtx_loadDictionary_byReference(ress.cctx, ress.dict.dictBuffer, ress.dict.dictBufferSize) );
} }
return ress; return ress;
@@ -1126,11 +1129,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
static void FIO_freeCResources(const cRess_t* const ress) static void FIO_freeCResources(const cRess_t* const ress)
{ {
if (!ress->mmapDict) { FIO_freeDict(&(ress->dict));
free(ress->dictBuffer);
} else {
FIO_munmapDictBuffer(ress->dictBuffer, ress->dictBufferSize);
}
AIO_WritePool_free(ress->writeCtx); AIO_WritePool_free(ress->writeCtx);
AIO_ReadPool_free(ress->readCtx); AIO_ReadPool_free(ress->readCtx);
ZSTD_freeCStream(ress->cctx); /* never fails */ ZSTD_freeCStream(ress->cctx); /* never fails */
@@ -2131,17 +2130,16 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
* Decompression * Decompression
***************************************************************************/ ***************************************************************************/
typedef struct { typedef struct {
void* dictBuffer; FIO_Dict_t dict;
size_t dictBufferSize;
ZSTD_DStream* dctx; ZSTD_DStream* dctx;
WritePoolCtx_t *writeCtx; WritePoolCtx_t *writeCtx;
ReadPoolCtx_t *readCtx; ReadPoolCtx_t *readCtx;
int mmapDict;
} dRess_t; } dRess_t;
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
{ {
int mmapDict = prefs->mmapDict; int useMMap = prefs->mmapDict == ZSTD_ps_enable;
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
stat_t statbuf; stat_t statbuf;
dRess_t ress; dRess_t ress;
memset(&ress, 0, sizeof(ress)); memset(&ress, 0, sizeof(ress));
@@ -2150,12 +2148,12 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
if (prefs->patchFromMode){ if (prefs->patchFromMode){
U64 const dictSize = UTIL_getFileSizeStat(&statbuf); U64 const dictSize = UTIL_getFileSizeStat(&statbuf);
mmapDict |= dictSize > prefs->memLimit; useMMap |= dictSize > prefs->memLimit;
FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */); FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
} }
/* Allocation */ /* Allocation */
ress.mmapDict = mmapDict; ress.dict.dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
ress.dctx = ZSTD_createDStream(); ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL) if (ress.dctx==NULL)
EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
@@ -2163,18 +2161,18 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag)); CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag));
/* dictionary */ /* dictionary */
{ if (!mmapDict) { { if (ress.dict.dictBufferType == FIO_mallocDict) {
ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs, &statbuf); ress.dict.dictBufferSize = FIO_createDictBuffer(&ress.dict.dictBuffer, dictFileName, prefs, &statbuf);
} else { } else {
ress.dictBufferSize = FIO_createDictBufferMMap(&ress.dictBuffer, dictFileName, prefs, &statbuf); ress.dict.dictBufferSize = FIO_createDictBufferMMap(&ress.dict.dictBuffer, dictFileName, prefs, &statbuf);
} }
CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) ); CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
if (prefs->patchFromMode){ if (prefs->patchFromMode){
CHECK(ZSTD_DCtx_refPrefix(ress.dctx, ress.dictBuffer, ress.dictBufferSize)); CHECK(ZSTD_DCtx_refPrefix(ress.dctx, ress.dict.dictBuffer, ress.dict.dictBufferSize));
} else { } else {
CHECK(ZSTD_DCtx_loadDictionary_byReference(ress.dctx, ress.dictBuffer, ress.dictBufferSize)); CHECK(ZSTD_DCtx_loadDictionary_byReference(ress.dctx, ress.dict.dictBuffer, ress.dict.dictBufferSize));
} }
} }
@@ -2185,11 +2183,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
static void FIO_freeDResources(dRess_t ress) static void FIO_freeDResources(dRess_t ress)
{ {
if (!ress.mmapDict) { FIO_freeDict(&(ress.dict));
free(ress.dictBuffer);
} else {
FIO_munmapDictBuffer(ress.dictBuffer, ress.dictBufferSize);
}
CHECK( ZSTD_freeDStream(ress.dctx) ); CHECK( ZSTD_freeDStream(ress.dctx) );
AIO_WritePool_free(ress.writeCtx); AIO_WritePool_free(ress.writeCtx);
AIO_ReadPool_free(ress.readCtx); AIO_ReadPool_free(ress.readCtx);

2
programs/fileio.h
View File

@@ -106,7 +106,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs); void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value); void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value); void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
void FIO_setMMapDict(FIO_prefs_t* const prefs, int value); void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_paramSwitch_e value);
/* FIO_ctx_t functions */ /* FIO_ctx_t functions */
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);

10
programs/fileio_types.h
View File

@@ -69,7 +69,15 @@ typedef struct FIO_prefs_s {
int contentSize; int contentSize;
int allowBlockDevices; int allowBlockDevices;
int passThrough; int passThrough;
int mmapDict; ZSTD_paramSwitch_e mmapDict;
} FIO_prefs_t; } FIO_prefs_t;
typedef enum {FIO_mallocDict, FIO_mmapDict} FIO_dictBufferType_t;
typedef struct {
void* dictBuffer;
size_t dictBufferSize;
FIO_dictBufferType_t dictBufferType;
} FIO_Dict_t;
#endif /* FILEIO_TYPES_HEADER */ #endif /* FILEIO_TYPES_HEADER */

7
programs/zstdcli.c
View File

@@ -851,8 +851,8 @@ int main(int argCount, const char* argv[])
showDefaultCParams = 0, showDefaultCParams = 0,
ultra=0, ultra=0,
contentSize=1, contentSize=1,
removeSrcFile=0, removeSrcFile=0;
mmapDict=0; ZSTD_paramSwitch_e mmapDict=ZSTD_ps_auto;
ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto; ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
FIO_compressionType_t cType = FIO_zstdCompression; FIO_compressionType_t cType = FIO_zstdCompression;
unsigned nbWorkers = 0; unsigned nbWorkers = 0;
@@ -986,7 +986,8 @@ int main(int argCount, const char* argv[])
if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; } if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; cType = FIO_zstdCompression; continue; } if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; cType = FIO_zstdCompression; continue; }
if (!strcmp(argument, "--mmap-dict")) { mmapDict = 1; continue; } if (!strcmp(argument, "--mmap-dict")) { mmapDict = ZSTD_ps_enable; continue; }
if (!strcmp(argument, "--no-mmap-dict")) { mmapDict = ZSTD_ps_disable; continue; }
#ifdef ZSTD_GZCOMPRESS #ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; cType = FIO_gzipCompression; continue; } if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; cType = FIO_gzipCompression; continue; }
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */ if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */