size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);start a new compression job, using same parameters from previous job. - This is typically useful to skip dictionary loading stage, since it will re-use it in-place.. + This is typically useful to skip dictionary loading stage, since it will re-use it in-place. Note that zcs must be init at least once before using ZSTD_resetCStream(). If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. - @return : 0, or an error code (which can be tested using ZSTD_isError()) + @return : 0, or an error code (which can be tested using ZSTD_isError()) +
typedef struct {
- unsigned long long ingested;
- unsigned long long consumed;
- unsigned long long produced;
- unsigned currentJobID;
+ unsigned long long ingested; /* nb input bytes read and buffered */
+ unsigned long long consumed; /* nb input bytes actually compressed */
+ unsigned long long produced; /* nb of compressed bytes generated and buffered */
+ unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
+ unsigned currentJobID; /* MT only : latest started job nb */
+ unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */
} ZSTD_frameProgression;
size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); +Tell how many bytes are ready to be flushed immediately. + Useful for multithreading scenarios (nbWorkers >= 1). + Probe the oldest active job, defined as oldest job not yet entirely flushed, + and check its output buffer. + @return : amount of data stored in oldest job and ready to be flushed immediately. + if @return == 0, it means either : + + there is no active job (could be checked with ZSTD_frameProgression()), or + + oldest job is still actively compressing data, + but everything it has produced has also been flushed so far, + therefore flushing speed is currently limited by production speed of oldest job + irrespective of the speed of concurrent newer jobs. + +
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); /* obsolete : this API will be removed in a future version */
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
@@ -1015,9 +1039,13 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx,
typedef enum {
- ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal conditions */
- ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
- ZSTD_e_end /* flush any remaining data and close current frame. Any additional data starts a new frame. */
+ ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+ ZSTD_e_flush, /* flush any data provided so far,
+ * it creates (at least) one new block, that can be decoded immediately on reception;
+ * frame will continue: any future data can still reference previously compressed data, improving compression. */
+ ZSTD_e_end /* flush any remaining data and close current frame.
+ * any additional data starts a new frame.
+ * each frame is independent (does not reference any content from previous frame). */
} ZSTD_EndDirective;
size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
diff --git a/programs/util.c b/programs/util.c
index 510576448..dfe2c8ab2 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -127,6 +127,196 @@ U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbF
return error ? UTIL_FILESIZE_UNKNOWN : total;
}
+ #ifdef _WIN32
+int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+{
+ char* path;
+ int dirLength, fnameLength, pathLength, nbFiles = 0;
+ WIN32_FIND_DATAA cFile;
+ HANDLE hFile;
+
+ dirLength = (int)strlen(dirName);
+ path = (char*) malloc(dirLength + 3);
+ if (!path) return 0;
+
+ memcpy(path, dirName, dirLength);
+ path[dirLength] = '\\';
+ path[dirLength+1] = '*';
+ path[dirLength+2] = 0;
+
+ hFile=FindFirstFileA(path, &cFile);
+ if (hFile == INVALID_HANDLE_VALUE) {
+ UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
+ return 0;
+ }
+ free(path);
+
+ do {
+ fnameLength = (int)strlen(cFile.cFileName);
+ path = (char*) malloc(dirLength + fnameLength + 2);
+ if (!path) { FindClose(hFile); return 0; }
+ memcpy(path, dirName, dirLength);
+ path[dirLength] = '\\';
+ memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
+ pathLength = dirLength+1+fnameLength;
+ path[pathLength] = 0;
+ if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+ if (strcmp (cFile.cFileName, "..") == 0 ||
+ strcmp (cFile.cFileName, ".") == 0) continue;
+
+ nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
+ if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
+ }
+ else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) {
+ if (*bufStart + *pos + pathLength >= *bufEnd) {
+ ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+ *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+ *bufEnd = *bufStart + newListSize;
+ if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
+ }
+ if (*bufStart + *pos + pathLength < *bufEnd) {
+ strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
+ *pos += pathLength + 1;
+ nbFiles++;
+ }
+ }
+ free(path);
+ } while (FindNextFileA(hFile, &cFile));
+
+ FindClose(hFile);
+ return nbFiles;
+}
+
+#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
+
+int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+{
+ DIR *dir;
+ struct dirent *entry;
+ char* path;
+ int dirLength, fnameLength, pathLength, nbFiles = 0;
+
+ if (!(dir = opendir(dirName))) {
+ UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
+ return 0;
+ }
+
+ dirLength = (int)strlen(dirName);
+ errno = 0;
+ while ((entry = readdir(dir)) != NULL) {
+ if (strcmp (entry->d_name, "..") == 0 ||
+ strcmp (entry->d_name, ".") == 0) continue;
+ fnameLength = (int)strlen(entry->d_name);
+ path = (char*) malloc(dirLength + fnameLength + 2);
+ if (!path) { closedir(dir); return 0; }
+ memcpy(path, dirName, dirLength);
+
+ path[dirLength] = '/';
+ memcpy(path+dirLength+1, entry->d_name, fnameLength);
+ pathLength = dirLength+1+fnameLength;
+ path[pathLength] = 0;
+
+ if (!followLinks && UTIL_isLink(path)) {
+ UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
+ continue;
+ }
+
+ if (UTIL_isDirectory(path)) {
+ nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
+ if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+ } else {
+ if (*bufStart + *pos + pathLength >= *bufEnd) {
+ ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
+ *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+ *bufEnd = *bufStart + newListSize;
+ if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
+ }
+ if (*bufStart + *pos + pathLength < *bufEnd) {
+ strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
+ *pos += pathLength + 1;
+ nbFiles++;
+ }
+ }
+ free(path);
+ errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
+ }
+
+ if (errno != 0) {
+ UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
+ free(*bufStart);
+ *bufStart = NULL;
+ }
+ closedir(dir);
+ return nbFiles;
+}
+
+#else
+
+int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+{
+ (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
+ UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
+ return 0;
+}
+
+#endif /* #ifdef _WIN32 */
+
+/*
+ * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
+ * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
+ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
+ * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
+ */
+const char**
+UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
+ char** allocatedBuffer, unsigned* allocatedNamesNb,
+ int followLinks)
+{
+ size_t pos;
+ unsigned i, nbFiles;
+ char* buf = (char*)malloc(LIST_SIZE_INCREASE);
+ char* bufend = buf + LIST_SIZE_INCREASE;
+ const char** fileTable;
+
+ if (!buf) return NULL;
+
+ for (i=0, pos=0, nbFiles=0; i= bufend) {
+ ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
+ buf = (char*)UTIL_realloc(buf, newListSize);
+ bufend = buf + newListSize;
+ if (!buf) return NULL;
+ }
+ if (buf + pos + len < bufend) {
+ strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
+ pos += len + 1;
+ nbFiles++;
+ }
+ } else {
+ nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
+ if (buf == NULL) return NULL;
+ } }
+
+ if (nbFiles == 0) { free(buf); return NULL; }
+
+ fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
+ if (!fileTable) { free(buf); return NULL; }
+
+ for (i=0, pos=0; i bufend) { free(buf); free((void*)fileTable); return NULL; }
+
+ *allocatedBuffer = buf;
+ *allocatedNamesNb = nbFiles;
+
+ return fileTable;
+}
+
/*-****************************************
* Time functions
******************************************/
diff --git a/programs/util.h b/programs/util.h
index 4e3586588..ca8dcfc4a 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -207,143 +207,14 @@ UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
return NULL;
}
+int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks);
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
-
-UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
-{
- char* path;
- int dirLength, fnameLength, pathLength, nbFiles = 0;
- WIN32_FIND_DATAA cFile;
- HANDLE hFile;
-
- dirLength = (int)strlen(dirName);
- path = (char*) malloc(dirLength + 3);
- if (!path) return 0;
-
- memcpy(path, dirName, dirLength);
- path[dirLength] = '\\';
- path[dirLength+1] = '*';
- path[dirLength+2] = 0;
-
- hFile=FindFirstFileA(path, &cFile);
- if (hFile == INVALID_HANDLE_VALUE) {
- UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
- return 0;
- }
- free(path);
-
- do {
- fnameLength = (int)strlen(cFile.cFileName);
- path = (char*) malloc(dirLength + fnameLength + 2);
- if (!path) { FindClose(hFile); return 0; }
- memcpy(path, dirName, dirLength);
- path[dirLength] = '\\';
- memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
- pathLength = dirLength+1+fnameLength;
- path[pathLength] = 0;
- if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
- if (strcmp (cFile.cFileName, "..") == 0 ||
- strcmp (cFile.cFileName, ".") == 0) continue;
-
- nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
- if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
- }
- else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) {
- if (*bufStart + *pos + pathLength >= *bufEnd) {
- ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
- *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
- *bufEnd = *bufStart + newListSize;
- if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
- }
- if (*bufStart + *pos + pathLength < *bufEnd) {
- strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
- *pos += pathLength + 1;
- nbFiles++;
- }
- }
- free(path);
- } while (FindNextFileA(hFile, &cFile));
-
- FindClose(hFile);
- return nbFiles;
-}
-
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
# define UTIL_HAS_CREATEFILELIST
# include /* opendir, readdir */
# include /* strerror, memcpy */
-
-UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
-{
- DIR *dir;
- struct dirent *entry;
- char* path;
- int dirLength, fnameLength, pathLength, nbFiles = 0;
-
- if (!(dir = opendir(dirName))) {
- UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
- return 0;
- }
-
- dirLength = (int)strlen(dirName);
- errno = 0;
- while ((entry = readdir(dir)) != NULL) {
- if (strcmp (entry->d_name, "..") == 0 ||
- strcmp (entry->d_name, ".") == 0) continue;
- fnameLength = (int)strlen(entry->d_name);
- path = (char*) malloc(dirLength + fnameLength + 2);
- if (!path) { closedir(dir); return 0; }
- memcpy(path, dirName, dirLength);
-
- path[dirLength] = '/';
- memcpy(path+dirLength+1, entry->d_name, fnameLength);
- pathLength = dirLength+1+fnameLength;
- path[pathLength] = 0;
-
- if (!followLinks && UTIL_isLink(path)) {
- UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
- continue;
- }
-
- if (UTIL_isDirectory(path)) {
- nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
- if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
- } else {
- if (*bufStart + *pos + pathLength >= *bufEnd) {
- ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
- *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
- *bufEnd = *bufStart + newListSize;
- if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
- }
- if (*bufStart + *pos + pathLength < *bufEnd) {
- strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
- *pos += pathLength + 1;
- nbFiles++;
- }
- }
- free(path);
- errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
- }
-
- if (errno != 0) {
- UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
- free(*bufStart);
- *bufStart = NULL;
- }
- closedir(dir);
- return nbFiles;
-}
-
#else
-
-UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
-{
- (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks;
- UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
- return 0;
-}
-
#endif /* #ifdef _WIN32 */
/*
@@ -352,56 +223,10 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
*/
-UTIL_STATIC const char**
+const char**
UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
char** allocatedBuffer, unsigned* allocatedNamesNb,
- int followLinks)
-{
- size_t pos;
- unsigned i, nbFiles;
- char* buf = (char*)malloc(LIST_SIZE_INCREASE);
- char* bufend = buf + LIST_SIZE_INCREASE;
- const char** fileTable;
-
- if (!buf) return NULL;
-
- for (i=0, pos=0, nbFiles=0; i= bufend) {
- ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
- buf = (char*)UTIL_realloc(buf, newListSize);
- bufend = buf + newListSize;
- if (!buf) return NULL;
- }
- if (buf + pos + len < bufend) {
- strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
- pos += len + 1;
- nbFiles++;
- }
- } else {
- nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
- if (buf == NULL) return NULL;
- } }
-
- if (nbFiles == 0) { free(buf); return NULL; }
-
- fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
- if (!fileTable) { free(buf); return NULL; }
-
- for (i=0, pos=0; i bufend) { free(buf); free((void*)fileTable); return NULL; }
-
- *allocatedBuffer = buf;
- *allocatedNamesNb = nbFiles;
-
- return fileTable;
-}
-
+ int followLinks);
UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
{
diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index c1896f8b8..ba69f1992 100644
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -88,7 +88,7 @@ fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLI
fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
-zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
+zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
$(CC) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@