lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2026-01-06 11:21:19 +03:00
Code Activity

cli : FIO_createDictBuffer() replaces FIO_loadFile()

Browse Source 
makes it more explicit that it allocates a buffer
and that it's meant to be used for dictionary.

Also : simplified function a bit,
now only works for dictionaries up to DICTSIZE_MAX
This commit is contained in:
Yann Collet
2017-04-11 14:41:02 -07:00
parent 1fa3b75369
commit 0e30059ba1
1 changed files with 11 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
programs/fileio.c
View File

@@ -76,7 +76,7 @@
#define CACHELINE 64
#define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */
#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */
#define FNSPACE 30
@@ -278,13 +278,13 @@ static FILE* FIO_openDstFile(const char* dstFileName)
}
/*! FIO_loadFile() :
* creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it,
* up to MAX_DICT_SIZE bytes.
* @return : loaded size
*/
static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
/*! FIO_createDictBuffer() :
* creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
{
FILE* fileHandle;
U64 fileSize;
@@ -296,14 +296,7 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
fileHandle = fopen(fileName, "rb");
if (fileHandle==0) EXM_THROW(31, "zstd: %s: %s", fileName, strerror(errno));
fileSize = UTIL_getFileSize(fileName);
if (fileSize > MAX_DICT_SIZE) {
int seekResult;
if (fileSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", fileName); /* avoid extreme cases */
DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", fileName, (U32)MAX_DICT_SIZE);
seekResult = fseek(fileHandle, (long int)(fileSize-MAX_DICT_SIZE), SEEK_SET); /* use end of file */
if (seekResult != 0) EXM_THROW(33, "zstd: %s: %s", fileName, strerror(errno));
fileSize = MAX_DICT_SIZE;
}
if (fileSize > DICTSIZE_MAX) EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */
*bufferPtr = malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "zstd: %s", strerror(errno));
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
@@ -356,7 +349,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
/* dictionary */
{ void* dictBuffer;
size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName);
size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName);
if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
params.fParams.contentSizeFlag = srcRegFile;
@@ -776,7 +769,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
/* dictionary */
{ void* dictBuffer;
size_t const dictBufferSize = FIO_loadFile(&dictBuffer, dictFileName);
size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName);
size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize);
if (ZSTD_isError(initError)) EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError));
free(dictBuffer);