1
0
mirror of https://github.com/facebook/zstd.git synced 2025-07-29 11:21:22 +03:00

refactor code to only use FileNamesTable*

This commit is contained in:
Yann Collet
2019-11-05 17:02:43 -08:00
parent 76b9e42b0b
commit b09f59390b
3 changed files with 393 additions and 420 deletions

View File

@ -335,6 +335,7 @@ UTIL_createFileNamesTable(const char** filenames, size_t tableSize, char* buf)
table->fileNames = filenames;
table->buf = buf;
table->tableSize = tableSize;
table->tableCapacity = tableSize;
return table;
}
@ -346,6 +347,23 @@ void UTIL_freeFileNamesTable(FileNamesTable* table)
free(table);
}
FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize)
{
const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable));
FileNamesTable* fnt;
if (fnTable==NULL) return NULL;
fnt = UTIL_createFileNamesTable(fnTable, tableSize, NULL);
fnt->tableSize = 0; /* the table is empty */
return fnt;
}
void UTIL_refFilename(FileNamesTable* fnt, const char* filename)
{
if (fnt->tableCapacity <= fnt->tableSize) abort();
fnt->fileNames[fnt->tableSize] = filename;
fnt->tableSize++;
}
static size_t getTotalTableSize(FileNamesTable* table)
{
size_t fnb = 0, totalSize = 0;
@ -568,27 +586,23 @@ const char* UTIL_getFileExtension(const char* infilename)
return extension;
}
/*
* UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
* and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
*/
const char**
UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
char** allocatedBuffer, unsigned* allocatedNamesNb,
int followLinks)
static FileNamesTable*
createFNT_fromFNT(FileNamesTable* fnt, int followLinks)
{
size_t pos;
unsigned i, nbFiles;
size_t const nbIfns = fnt->tableSize;
unsigned nbFiles;
const char** const inputNames = fnt->fileNames;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
if (!buf) return NULL;
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
if (!UTIL_isDirectory(inputNames[i])) {
size_t const len = strlen(inputNames[i]);
{ size_t ifnNb;
for (ifnNb=0, pos=0, nbFiles=0; ifnNb<nbIfns; ifnNb++) {
if (!UTIL_isDirectory(inputNames[ifnNb])) {
size_t const len = strlen(inputNames[ifnNb]);
if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
assert(newListSize >= 0);
@ -597,42 +611,41 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
if (!buf) return NULL;
}
if (buf + pos + len < bufend) {
memcpy(buf+pos, inputNames[i], len+1); /* including final \0 */
memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */
pos += len + 1;
nbFiles++;
}
} else {
nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks);
if (buf == NULL) return NULL;
} }
} } }
if (nbFiles == 0) { free(buf); return NULL; }
{ const char** const fileTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileTable));
if (!fileTable) { free(buf); return NULL; }
{ size_t ifnNb;
const char** const fileNamesTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileNamesTable));
if (!fileNamesTable) { free(buf); return NULL; }
for (i = 0, pos = 0; i < nbFiles; i++) {
fileTable[i] = buf + pos;
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
pos += strlen(fileTable[i]) + 1;
for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) {
fileNamesTable[ifnNb] = buf + pos;
if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; }
pos += strlen(fileNamesTable[ifnNb]) + 1;
}
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
return fileTable;
return UTIL_createFileNamesTable(fileNamesTable, nbFiles, buf);
}
}
void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
FileNamesTable*
UTIL_expandFileNamesTable(FileNamesTable* fnt, int followLinks)
{
if (allocatedBuffer) free(allocatedBuffer);
if (filenameTable) free((void*)filenameTable);
FileNamesTable* const newFNT = createFNT_fromFNT(fnt, followLinks);
UTIL_freeFileNamesTable(fnt);
return newFNT;
}
/*-****************************************
* count the number of physical cores
******************************************/

View File

@ -159,12 +159,12 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
/* do not define UTIL_HAS_CREATEFILELIST */
#endif /* #ifdef _WIN32 */
/*Note: tableSize denotes the total capacity of table*/
typedef struct
{
const char** fileNames;
char* buf;
size_t tableSize;
char* buf; /* fileNames are stored in this buffer (or are read-only) */
size_t tableSize; /* nb of fileNames */
size_t tableCapacity;
} FileNamesTable;
/*! UTIL_createFileNamesTable_fromFileName() :
@ -183,7 +183,6 @@ UTIL_createFileNamesTable_fromFileName(const char* inputFileName);
FileNamesTable*
UTIL_createFileNamesTable(const char** filenames, size_t tableSize, char* buf);
/*! UTIL_freeFileNamesTable() :
* This function is compatible with NULL argument and never fails.
*/
@ -197,19 +196,31 @@ FileNamesTable*
UTIL_concatenateTwoTables(FileNamesTable* table1, FileNamesTable* table2);
/*
* UTIL_createFileList() :
* takes a list of files and directories (@inputNames, @inputNamesNb),
* scans directories, and returns a new list of files (@return, @allocatedBuffer, @allocatedNamesNb).
* In case of error, UTIL_createFileList() returns NULL.
* After list's end of life, the structures should be freed with UTIL_freeFileList (@return, @allocatedBuffer).
/*! UTIL_expandFileNamesTable() :
* read names from @fnt, expand those corresponding to directories
* @return : an expanded FileNamesTable*, with only file names,
* or NULL in case of error.
* Note: the function takes ownership of fnt, and consumes it (free it)
*/
const char**
UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
char** allocatedBuffer, unsigned* allocatedNamesNb,
int followLinks);
FileNamesTable* UTIL_expandFileNamesTable(FileNamesTable* fnt, int followLinks);
void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer);
/*! UTIL_allocateFileNamesTable() :
* Allocates a table of const char*, to insert read-only names later on.
* The created FileNamesTable* doesn't hold a buffer.
* @return : FileNamesTable*, or NULL, if allocation fails.
*/
FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize);
/*! UTIL_refFilename() :
* Add a read-only name to reference into @fnt table.
* Since @filename is only referenced, its lifetime must outlive @fnt.
* This function never fails, but it can abort().
* Internal table must be large enough to reference a new member
* (capacity > size), otherwise the function will abort().
*/
void UTIL_refFilename(FileNamesTable* fnt, const char* filename);
/*-****************************************

View File

@ -585,10 +585,8 @@ int main(int argCount, const char* argv[])
int cLevelLast = -1000000000;
unsigned recursive = 0;
unsigned memLimit = 0;
size_t filenameTableSize = argCount;
const char** filenameTable = (const char**)malloc(filenameTableSize * sizeof(const char*)); /* argCount >= 1 */
char* tableBuf = NULL;
unsigned filenameIdx = 0;
FileNamesTable* filenames = UTIL_allocateFileNamesTable((size_t)argCount); /* argCount >= 1 */
FileNamesTable* file_of_names = UTIL_allocateFileNamesTable((size_t)argCount); /* argCount >= 1 */
const char* programName = argv[0];
const char* outFileName = NULL;
const char* outDirName = NULL;
@ -601,11 +599,6 @@ int main(int argCount, const char* argv[])
size_t srcSizeHint = 0;
int dictCLevel = g_defaultDictCLevel;
unsigned dictSelect = g_defaultSelectivityLevel;
#ifdef UTIL_HAS_CREATEFILELIST
const char** extendedFileList = NULL;
char* fileNamesBuf = NULL;
unsigned fileNamesNb;
#endif
#ifndef ZSTD_NODICT
ZDICT_cover_params_t coverParams = defaultCoverParams();
ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams();
@ -620,8 +613,7 @@ int main(int argCount, const char* argv[])
/* init */
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)memLimit; /* not used when ZSTD_NODECOMPRESS set */
if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
filenameTable[0] = stdinmark;
if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAY("zstd: allocation error \n"); exit(1); }
g_displayOut = stderr;
cLevel = init_cLevel();
programName = lastNameFromPath(programName);
@ -653,15 +645,16 @@ int main(int argCount, const char* argv[])
const char* argument = argv[argNb];
if (!argument) continue; /* Protection if argument empty */
if (nextArgumentsAreFiles==0) {
if (nextArgumentsAreFiles) {
UTIL_refFilename(filenames, argument);
continue;
}
/* "-" means stdin/stdout */
if (!strcmp(argument, "-")){
if (!filenameIdx) {
filenameIdx=1, filenameTable[0]=stdinmark;
outFileName=stdoutmark;
g_displayLevel-=(g_displayLevel==2);
UTIL_refFilename(filenames, stdinmark);
continue;
} }
}
/* Decode commands (note : aggregated commands are allowed) */
if (argument[0]=='-') {
@ -801,55 +794,10 @@ int main(int argCount, const char* argv[])
#endif
if (longCommandWArg(&argument, "--filelist=")) {
/* note : in theory, it's better to just store the arguments at this stage,
* and only start to load & interpret the file after command line is parsed.
* For a single file, it would be easy to just store its name here, and parse later.
* However, this implementation makes it possible to read multiple files.
* An equivalent will have to be able to store multiple file names.
*/
FileNamesTable* extendedTable;
FileNamesTable* curTable;
FileNamesTable* concatenatedTables;
if (!UTIL_fileExist(argument) || !UTIL_isRegularFile(argument)){
DISPLAYLEVEL(1, "[ERROR] wrong fileName: %s\n", argument);
CLEAN_RETURN(badusage(programName));
}
extendedTable = UTIL_createFileNamesTable_fromFileName(argument);
if (!extendedTable) {
CLEAN_RETURN(badusage(programName));
}
filenameTable[filenameIdx] = NULL; /* marking end of table */
filenameIdx += (unsigned) extendedTable->tableSize;
curTable = UTIL_createFileNamesTable(filenameTable, filenameTableSize, tableBuf);
if (!curTable) {
UTIL_freeFileNamesTable(extendedTable);
CLEAN_RETURN(badusage(programName));
}
concatenatedTables = UTIL_concatenateTwoTables(curTable, extendedTable);
if (!concatenatedTables) {
UTIL_freeFileNamesTable(curTable);
UTIL_freeFileNamesTable(extendedTable);
CLEAN_RETURN(badusage(programName));
}
/* transfer ownership */
filenameTable = concatenatedTables->fileNames;
filenameTableSize = concatenatedTables->tableSize;
tableBuf = concatenatedTables->buf;
concatenatedTables->fileNames = NULL;
concatenatedTables->tableSize = 0;
concatenatedTables->buf = NULL;
UTIL_freeFileNamesTable(concatenatedTables);
UTIL_refFilename(file_of_names, argument);
continue;
}
/* fall-through, will trigger bad_usage() later on */
}
@ -1007,8 +955,6 @@ int main(int argCount, const char* argv[])
continue;
}
} /* if (nextArgumentIsAFile==0) */
if (nextEntryIsDictionary) {
nextEntryIsDictionary = 0;
lastCommand = 0;
@ -1031,8 +977,8 @@ int main(int argCount, const char* argv[])
continue;
}
/* add filename to list */
filenameTable[filenameIdx++] = argument;
/* none of the above : add filename to list */
UTIL_refFilename(filenames, argument);
}
if (lastCommand) { /* forgotten argument */
@ -1056,37 +1002,47 @@ int main(int argCount, const char* argv[])
#ifdef UTIL_HAS_CREATEFILELIST
g_utilDisplayLevel = g_displayLevel;
if (!followLinks) {
unsigned u;
for (u=0, fileNamesNb=0; u<filenameIdx; u++) {
if (UTIL_isLink(filenameTable[u])
unsigned u, fileNamesNb;
unsigned const nbFilenames = (unsigned)filenames->tableSize;
for (u=0, fileNamesNb=0; u<nbFilenames; u++) {
if (UTIL_isLink(filenames->fileNames[u])
#ifndef _MSC_VER
&& !UTIL_isFIFO(filenameTable[u])
&& !UTIL_isFIFO(filenames->fileNames[u])
#endif /* _MSC_VER */
) {
DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenames->fileNames[u]);
} else {
filenameTable[fileNamesNb++] = filenameTable[u];
filenames->fileNames[fileNamesNb++] = filenames->fileNames[u];
} }
if (fileNamesNb == 0 && filenameIdx > 0)
if (fileNamesNb == 0 && nbFilenames > 0) /* all names are eliminated */
CLEAN_RETURN(1);
filenames->tableSize = fileNamesNb;
} /* if (!followLinks) */
/* read names from a file */
if (file_of_names->tableSize) {
size_t const nbFileLists = file_of_names->tableSize;
size_t flNb;
for (flNb=0; flNb < nbFileLists; flNb++) {
FileNamesTable* const fnt = UTIL_createFileNamesTable_fromFileName(file_of_names->fileNames[flNb]);
if (fnt==NULL) {
DISPLAYLEVEL(1, "zstd: error reading %s \n", file_of_names->fileNames[flNb]);
CLEAN_RETURN(1);
filenameIdx = fileNamesNb;
}
filenames = UTIL_concatenateTwoTables(filenames, fnt);
}
}
if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks);
if (extendedFileList) {
unsigned u;
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
free((void*)filenameTable);
filenameTable = extendedFileList;
filenameIdx = fileNamesNb;
} }
filenames = UTIL_expandFileNamesTable(filenames, followLinks);
}
#else
(void)followLinks;
#endif
if (operation == zom_list) {
#ifndef ZSTD_NODECOMPRESS
int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
int const ret = FIO_listMultipleFiles((unsigned)filenames->tableSize, filenames->fileNames, g_displayLevel);
CLEAN_RETURN(ret);
#else
DISPLAY("file information is not supported \n");
@ -1117,18 +1073,18 @@ int main(int argCount, const char* argv[])
if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel)
DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (filenameIdx) {
if (filenames->tableSize > 0) {
if(separateFiles) {
unsigned i;
for(i = 0; i < filenameIdx; i++) {
for(i = 0; i < filenames->tableSize; i++) {
int c;
DISPLAYLEVEL(3, "Benchmarking %s \n", filenameTable[i]);
DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
for(c = cLevel; c <= cLevelLast; c++) {
BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
@ -1152,18 +1108,18 @@ int main(int argCount, const char* argv[])
int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = (unsigned)nbWorkers;
coverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (unsigned)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize);
} else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = (unsigned)nbWorkers;
fastCoverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (unsigned)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize);
} else {
ZDICT_legacy_params_t dictParams;
memset(&dictParams, 0, sizeof(dictParams));
dictParams.selectivityLevel = dictSelect;
dictParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (unsigned)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0);
}
#else
(void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */
@ -1178,16 +1134,16 @@ int main(int argCount, const char* argv[])
#endif
/* No input filename ==> use stdin and stdout */
filenameIdx += !filenameIdx; /* filenameTable[0] is stdin by default */
if (!strcmp(filenameTable[0], stdinmark) && !outFileName)
if (filenames->tableSize == 0) UTIL_refFilename(filenames, stdinmark);
if (!strcmp(filenames->fileNames[0], stdinmark) && !outFileName)
outFileName = stdoutmark; /* when input is stdin, default output is stdout */
/* Check if input/output defined as console; trigger an error in this case */
if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) )
if (!strcmp(filenames->fileNames[0], stdinmark) && IS_CONSOLE(stdin) )
CLEAN_RETURN(badusage(programName));
if ( outFileName && !strcmp(outFileName, stdoutmark)
&& IS_CONSOLE(stdout)
&& !strcmp(filenameTable[0], stdinmark)
&& !strcmp(filenames->fileNames[0], stdinmark)
&& !forceStdout
&& operation!=zom_decompress )
CLEAN_RETURN(badusage(programName));
@ -1202,8 +1158,8 @@ int main(int argCount, const char* argv[])
#endif
/* No status message in pipe mode (stdin - stdout) or multi-files mode */
if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
if ((filenameIdx>1) & (g_displayLevel==2)) g_displayLevel=1;
if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
if ((filenames->tableSize > 1) & (g_displayLevel==2)) g_displayLevel=1;
/* IO Stream/File */
FIO_setNotificationLevel(g_displayLevel);
@ -1228,10 +1184,10 @@ int main(int argCount, const char* argv[])
if (adaptMin > cLevel) cLevel = adaptMin;
if (adaptMax < cLevel) cLevel = adaptMax;
if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
if ((filenames->tableSize==1) && outFileName)
operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
else
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");
@ -1246,10 +1202,10 @@ int main(int argCount, const char* argv[])
}
}
FIO_setMemLimit(prefs, memLimit);
if (filenameIdx==1 && outFileName) {
operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
if (filenames->tableSize == 1 && outFileName) {
operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName);
} else {
operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName);
operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, dictFileName);
}
#else
DISPLAY("Decompression not supported \n");
@ -1258,16 +1214,9 @@ int main(int argCount, const char* argv[])
_end:
FIO_freePreferences(prefs);
free(tableBuf);
if (main_pause) waitEnter();
#ifdef UTIL_HAS_CREATEFILELIST
if (extendedFileList)
UTIL_freeFileList(extendedFileList, fileNamesBuf);
else
#endif
free((void*)filenameTable);
UTIL_freeFileNamesTable(filenames);
UTIL_freeFileNamesTable(file_of_names);
return operationResult;
}