lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-05 19:15:58 +03:00
Code Activity

Merge pull request #1834 from facebook/winFix

Browse Source 
Windows fixes
This commit is contained in:
Yann Collet
2019-10-21 13:45:17 -07:00
committed by GitHub
parent cf725630a6 bad35bd307
commit 6cf04c0344
13 changed files with 331 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.travis.yml
View File

@@ -54,7 +54,10 @@ matrix:
- name: Trusty (clang-3.8 + MSan + Test Zstd) - name: Trusty (clang-3.8 + MSan + Test Zstd)
script: script:
- make clang38install - make clang38install
- CC=clang-3.8 make clean msan-test-zstd # External libraries must be turned off when using MSAN tests,
# because they are not msan-instrumented,
# so any data coming from these libraries is always considered "uninitialized"
- CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
- name: Trusty (Minimal Decompressor Macros) - name: Trusty (Minimal Decompressor Macros)
script: script:

17
appveyor.yml
View File

@@ -1,3 +1,7 @@
# Following tests are run _only_ on master branch
# To reproduce these tests, it's possible to push into a branch `appveyorTest`
# or a branch `visual*`, they will intentionnally trigger `master` tests
- -
version: 1.0.{build} version: 1.0.{build}
branches: branches:
@@ -176,6 +180,11 @@
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
) )
# The following tests are for regular pushes
# into `dev` or some feature branch
# There run less tests, for shorter feedback loop
- -
version: 1.0.{build} version: 1.0.{build}
environment: environment:
@@ -249,3 +258,11 @@
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
) )
test_script:
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
- if [%HOST%]==[mingw] (
set "CC=%COMPILER%" &&
make check
)

24
doc/educational_decoder/Makefile
View File

@@ -7,8 +7,15 @@
# in the COPYING file in the root directory of this source tree). # in the COPYING file in the root directory of this source tree).
# ################################################################ # ################################################################
ZSTD ?= zstd # requires zstd installation on local system ZSTD ?= zstd # note: requires zstd installation on local system
UNAME?= $(shell uname)
ifeq ($(UNAME), SunOS)
DIFF ?= gdiff
else
DIFF ?= diff DIFF ?= diff
endif
HARNESS_FILES=*.c HARNESS_FILES=*.c
MULTITHREAD_LDFLAGS = -pthread MULTITHREAD_LDFLAGS = -pthread
@@ -29,26 +36,27 @@ harness: $(HARNESS_FILES)
$(CC) $(FLAGS) $^ -o $@ $(CC) $(FLAGS) $^ -o $@
clean: clean:
@$(RM) -f harness @$(RM) harness
@$(RM) -rf harness.dSYM @$(RM) -rf harness.dSYM # MacOS specific
test: harness test: harness
# #
# Testing single-file decompression with educational decoder # Testing single-file decompression with educational decoder
# #
@$(ZSTD) README.md -o tmp.zst @$(ZSTD) -f README.md -o tmp.zst
@./harness tmp.zst tmp @./harness tmp.zst tmp
@$(DIFF) -s tmp README.md @$(DIFF) -s tmp README.md
@$(RM) -f tmp* @$(RM) tmp*
# #
# Testing dictionary decompression with education decoder # Testing dictionary decompression with education decoder
# #
# note : files are presented multiple for training, to reach minimum threshold # note : files are presented multiple for training, to reach minimum threshold
@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \ @$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
harness.c zstd_decompress.c zstd_decompress.h README.md \ harness.c zstd_decompress.c zstd_decompress.h README.md \
harness.c zstd_decompress.c zstd_decompress.h README.md harness.c zstd_decompress.c zstd_decompress.h README.md \
@$(ZSTD) -D dictionary README.md -o tmp.zst -o dictionary
@$(ZSTD) -f README.md -D dictionary -o tmp.zst
@./harness tmp.zst tmp dictionary @./harness tmp.zst tmp dictionary
@$(DIFF) -s tmp README.md @$(DIFF) -s tmp README.md
@$(RM) -f tmp* dictionary @$(RM) tmp* dictionary
@$(MAKE) clean @$(MAKE) clean

83
doc/educational_decoder/harness.c
View File

@@ -21,88 +21,90 @@ typedef unsigned char u8;
// Protect against allocating too much memory for output // Protect against allocating too much memory for output
#define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024) #define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024)
u8 *input; static size_t read_file(const char *path, u8 **ptr)
u8 *output; {
u8 *dict; FILE* const f = fopen(path, "rb");
size_t read_file(const char *path, u8 **ptr) {
FILE *f = fopen(path, "rb");
if (!f) { if (!f) {
fprintf(stderr, "failed to open file %s\n", path); fprintf(stderr, "failed to open file %s \n", path);
exit(1); exit(1);
} }
fseek(f, 0L, SEEK_END); fseek(f, 0L, SEEK_END);
size_t size = (size_t)ftell(f); size_t const size = (size_t)ftell(f);
rewind(f); rewind(f);
*ptr = malloc(size); *ptr = malloc(size);
if (!ptr) { if (!ptr) {
fprintf(stderr, "failed to allocate memory to hold %s\n", path); fprintf(stderr, "failed to allocate memory to hold %s \n", path);
exit(1); exit(1);
} }
size_t pos = 0; size_t const read = fread(*ptr, 1, size, f);
while (!feof(f)) { if (read != size) { /* must read everything in one pass */
size_t read = fread(&(*ptr)[pos], 1, size, f); fprintf(stderr, "error while reading file %s \n", path);
if (ferror(f)) { exit(1);
fprintf(stderr, "error while reading file %s\n", path);
exit(1);
}
pos += read;
} }
fclose(f); fclose(f);
return pos; return read;
} }
void write_file(const char *path, const u8 *ptr, size_t size) { static void write_file(const char *path, const u8 *ptr, size_t size)
FILE *f = fopen(path, "wb"); {
FILE* const f = fopen(path, "wb");
if (!f) {
fprintf(stderr, "failed to open file %s \n", path);
exit(1);
}
size_t written = 0; size_t written = 0;
while (written < size) { while (written < size) {
written += fwrite(&ptr[written], 1, size, f); written += fwrite(ptr+written, 1, size, f);
if (ferror(f)) { if (ferror(f)) {
fprintf(stderr, "error while writing file %s\n", path); fprintf(stderr, "error while writing file %s\n", path);
exit(1); exit(1);
} } }
}
fclose(f); fclose(f);
} }
int main(int argc, char **argv) { int main(int argc, char **argv)
{
if (argc < 3) { if (argc < 3) {
fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary]\n", fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary] \n",
argv[0]); argv[0]);
return 1; return 1;
} }
size_t input_size = read_file(argv[1], &input); u8* input;
size_t const input_size = read_file(argv[1], &input);
u8* dict = NULL;
size_t dict_size = 0; size_t dict_size = 0;
if (argc >= 4) { if (argc >= 4) {
dict_size = read_file(argv[3], &dict); dict_size = read_file(argv[3], &dict);
} }
size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size); size_t out_capacity = ZSTD_get_decompressed_size(input, input_size);
if (decompressed_size == (size_t)-1) { if (out_capacity == (size_t)-1) {
decompressed_size = MAX_COMPRESSION_RATIO * input_size; out_capacity = MAX_COMPRESSION_RATIO * input_size;
fprintf(stderr, "WARNING: Compressed data does not contain " fprintf(stderr, "WARNING: Compressed data does not contain "
"decompressed size, going to assume the compression " "decompressed size, going to assume the compression "
"ratio is at most %d (decompressed size of at most " "ratio is at most %d (decompressed size of at most "
"%zu)\n", "%u) \n",
MAX_COMPRESSION_RATIO, decompressed_size); MAX_COMPRESSION_RATIO, (unsigned)out_capacity);
} }
if (decompressed_size > MAX_OUTPUT_SIZE) { if (out_capacity > MAX_OUTPUT_SIZE) {
fprintf(stderr, fprintf(stderr,
"Required output size too large for this implementation\n"); "Required output size too large for this implementation \n");
return 1; return 1;
} }
output = malloc(decompressed_size);
u8* const output = malloc(out_capacity);
if (!output) { if (!output) {
fprintf(stderr, "failed to allocate memory\n"); fprintf(stderr, "failed to allocate memory \n");
return 1; return 1;
} }
@@ -110,16 +112,17 @@ int main(int argc, char **argv) {
if (dict) { if (dict) {
parse_dictionary(parsed_dict, dict, dict_size); parse_dictionary(parsed_dict, dict, dict_size);
} }
size_t decompressed = size_t const decompressed_size =
ZSTD_decompress_with_dict(output, decompressed_size, ZSTD_decompress_with_dict(output, out_capacity,
input, input_size, parsed_dict); input, input_size,
parsed_dict);
free_dictionary(parsed_dict); free_dictionary(parsed_dict);
write_file(argv[2], output, decompressed); write_file(argv[2], output, decompressed_size);
free(input); free(input);
free(output); free(output);
free(dict); free(dict);
input = output = dict = NULL; return 0;
} }

2
lib/compress/zstd_cwksp.h
View File

@@ -490,7 +490,7 @@ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
} }
MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
} }
MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {

262
programs/fileio.c
View File

@@ -284,9 +284,10 @@ void FIO_addAbortHandler()
/*-************************************* /*-*************************************
* Parameters: Typedefs * Parameters: FIO_prefs_t
***************************************/ ***************************************/
/* typedef'd to FIO_prefs_t within fileio.h */
struct FIO_prefs_s { struct FIO_prefs_s {
/* Algorithm preferences */ /* Algorithm preferences */
@@ -308,6 +309,7 @@ struct FIO_prefs_s {
size_t streamSrcSize; size_t streamSrcSize;
size_t targetCBlockSize; size_t targetCBlockSize;
int srcSizeHint; int srcSizeHint;
int testMode;
ZSTD_literalCompressionMode_e literalCompressionMode; ZSTD_literalCompressionMode_e literalCompressionMode;
/* IO preferences */ /* IO preferences */
@@ -355,6 +357,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->streamSrcSize = 0; ret->streamSrcSize = 0;
ret->targetCBlockSize = 0; ret->targetCBlockSize = 0;
ret->srcSizeHint = 0; ret->srcSizeHint = 0;
ret->testMode = 0;
ret->literalCompressionMode = ZSTD_lcm_auto; ret->literalCompressionMode = ZSTD_lcm_auto;
return ret; return ret;
} }
@@ -435,6 +438,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) {
prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint);
} }
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
prefs->testMode = (testMode!=0);
}
void FIO_setLiteralCompressionMode( void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs, FIO_prefs_t* const prefs,
ZSTD_literalCompressionMode_e mode) { ZSTD_literalCompressionMode_e mode) {
@@ -529,8 +536,12 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
/** FIO_openDstFile() : /** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL. * condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */ * @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName) static FILE*
FIO_openDstFile(FIO_prefs_t* const prefs,
const char* srcFileName, const char* dstFileName)
{ {
if (prefs->testMode) return NULL; /* do not open file in test mode */
assert(dstFileName != NULL); assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) { if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output \n"); DISPLAYLEVEL(4,"Using stdout for output \n");
@@ -555,10 +566,14 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
if (UTIL_isRegularFile(dstFileName)) { if (UTIL_isRegularFile(dstFileName)) {
/* Check if destination file already exists */ /* Check if destination file already exists */
FILE* const fCheck = fopen( dstFileName, "rb" ); FILE* const fCheck = fopen( dstFileName, "rb" );
#if !defined(_WIN32)
/* this test does not work on Windows :
* `NUL` and `nul` are detected as regular files */
if (!strcmp(dstFileName, nulmark)) { if (!strcmp(dstFileName, nulmark)) {
EXM_THROW(40, "%s is unexpectedly categorized as a regular file", EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
dstFileName); dstFileName);
} }
#endif
if (fCheck != NULL) { /* dst file exists, authorization prompt */ if (fCheck != NULL) { /* dst file exists, authorization prompt */
fclose(fCheck); fclose(fCheck);
if (!prefs->overwrite) { if (!prefs->overwrite) {
@@ -648,7 +663,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
DISPLAY("Unable to malloc new str array, not checking for name collisions\n"); DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
return 1; return 1;
} }
for (u = 0; u < nbFiles; ++u) { for (u = 0; u < nbFiles; ++u) {
filename = strrchr(filenameTable[u], c[0]); filename = strrchr(filenameTable[u], c[0]);
if (filename == NULL) { if (filename == NULL) {
@@ -671,56 +686,50 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
return 0; return 0;
} }
static const char*
extractFilename(const char* path, char separator)
{
const char* search = strrchr(path, separator);
if (search == NULL) return path;
return search+1;
}
/* FIO_createFilename_fromOutDir() : /* FIO_createFilename_fromOutDir() :
* Takes a source file name and specified output directory, and * Takes a source file name and specified output directory, and
* allocates memory for and returns a pointer to final path. * allocates memory for and returns a pointer to final path.
* This function never returns an error (it may abort() in case of pb) * This function never returns an error (it may abort() in case of pb)
*/ */
static char* static char*
FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, const size_t suffixLen) FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen)
{ {
const char* c, *filenameBegin; const char* filenameStart;
char* filename, *result; char separator;
size_t finalPathLen; char* result;
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
c = "\\"; separator = '\\';
#else #else
c = "/"; separator = '/';
#endif #endif
finalPathLen = strlen(outDirName); filenameStart = extractFilename(path, separator);
filenameBegin = strrchr(srcFilename, c[0]); #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
if (filenameBegin == NULL) { filenameStart = extractFilename(filenameStart, '/'); /* sometimes, '/' separator is also used on Windows (mingw+msys2) */
filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char)); #endif
if (!filename) {
EXM_THROW(30, "zstd: %s", strerror(errno));
}
strcpy(filename, srcFilename);
} else {
filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
if (!filename) {
EXM_THROW(30, "zstd: %s", strerror(errno));
}
strcpy(filename, filenameBegin+1);
}
finalPathLen += strlen(filename); result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1);
result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char));
if (!result) { if (!result) {
free(filename); EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno));
EXM_THROW(30, "zstd: %s", strerror(errno));
} }
strcpy(result, outDirName); memcpy(result, outDirName, strlen(outDirName));
if (outDirName[strlen(outDirName)-1] == c[0]) { if (outDirName[strlen(outDirName)-1] == separator) {
strcat(result, filename); memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart));
} else { } else {
strcat(result, c); memcpy(result + strlen(outDirName), &separator, 1);
strcat(result, filename); memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart));
} }
free(filename);
return result; return result;
} }
@@ -825,13 +834,12 @@ static void FIO_freeCResources(cRess_t ress)
#ifdef ZSTD_GZCOMPRESS #ifdef ZSTD_GZCOMPRESS
static unsigned long long static unsigned long long
FIO_compressGzFrame(cRess_t* ress, FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */
const char* srcFileName, U64 const srcFileSize, const char* srcFileName, U64 const srcFileSize,
int compressionLevel, U64* readsize) int compressionLevel, U64* readsize)
{ {
unsigned long long inFileSize = 0, outFileSize = 0; unsigned long long inFileSize = 0, outFileSize = 0;
z_stream strm; z_stream strm;
int ret;
if (compressionLevel > Z_BEST_COMPRESSION) if (compressionLevel > Z_BEST_COMPRESSION)
compressionLevel = Z_BEST_COMPRESSION; compressionLevel = Z_BEST_COMPRESSION;
@@ -840,11 +848,12 @@ FIO_compressGzFrame(cRess_t* ress,
strm.zfree = Z_NULL; strm.zfree = Z_NULL;
strm.opaque = Z_NULL; strm.opaque = Z_NULL;
ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, { int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
15 /* maxWindowLogSize */ + 16 /* gzip only */, 15 /* maxWindowLogSize */ + 16 /* gzip only */,
8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
if (ret != Z_OK) if (ret != Z_OK) {
EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
} }
strm.next_in = 0; strm.next_in = 0;
strm.avail_in = 0; strm.avail_in = 0;
@@ -852,6 +861,7 @@ FIO_compressGzFrame(cRess_t* ress,
strm.avail_out = (uInt)ress->dstBufferSize; strm.avail_out = (uInt)ress->dstBufferSize;
while (1) { while (1) {
int ret;
if (strm.avail_in == 0) { if (strm.avail_in == 0) {
size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
if (inSize == 0) break; if (inSize == 0) break;
@@ -862,32 +872,31 @@ FIO_compressGzFrame(cRess_t* ress,
ret = deflate(&strm, Z_NO_FLUSH); ret = deflate(&strm, Z_NO_FLUSH);
if (ret != Z_OK) if (ret != Z_OK)
EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret);
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out; { size_t const cSize = ress->dstBufferSize - strm.avail_out;
if (decompBytes) { if (cSize) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
EXM_THROW(73, "Write error : cannot write to output file : %s", strerror(errno)); EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno));
outFileSize += decompBytes; outFileSize += cSize;
strm.next_out = (Bytef*)ress->dstBuffer; strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize; strm.avail_out = (uInt)ress->dstBufferSize;
} } }
} if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
(unsigned)(inFileSize>>20), (unsigned)(inFileSize>>20),
(double)outFileSize/inFileSize*100) (double)outFileSize/inFileSize*100)
else } else {
DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
(unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
(double)outFileSize/inFileSize*100); (double)outFileSize/inFileSize*100);
} } }
while (1) { while (1) {
ret = deflate(&strm, Z_FINISH); int const ret = deflate(&strm, Z_FINISH);
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out; { size_t const cSize = ress->dstBufferSize - strm.avail_out;
if (decompBytes) { if (cSize) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
EXM_THROW(75, "Write error : %s", strerror(errno)); EXM_THROW(75, "Write error : %s ", strerror(errno));
outFileSize += decompBytes; outFileSize += cSize;
strm.next_out = (Bytef*)ress->dstBuffer; strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize; strm.avail_out = (uInt)ress->dstBufferSize;
} } } }
@@ -896,11 +905,11 @@ FIO_compressGzFrame(cRess_t* ress,
EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret);
} }
ret = deflateEnd(&strm); { int const ret = deflateEnd(&strm);
if (ret != Z_OK) if (ret != Z_OK) {
EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
} }
*readsize = inFileSize; *readsize = inFileSize;
return outFileSize; return outFileSize;
} }
#endif #endif
@@ -923,14 +932,14 @@ FIO_compressLzmaFrame(cRess_t* ress,
if (plain_lzma) { if (plain_lzma) {
lzma_options_lzma opt_lzma; lzma_options_lzma opt_lzma;
if (lzma_lzma_preset(&opt_lzma, compressionLevel)) if (lzma_lzma_preset(&opt_lzma, compressionLevel))
EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName); EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName);
ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
if (ret != LZMA_OK) if (ret != LZMA_OK)
EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
} else { } else {
ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
if (ret != LZMA_OK) if (ret != LZMA_OK)
EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
} }
strm.next_in = 0; strm.next_in = 0;
@@ -950,11 +959,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
ret = lzma_code(&strm, action); ret = lzma_code(&strm, action);
if (ret != LZMA_OK && ret != LZMA_STREAM_END) if (ret != LZMA_OK && ret != LZMA_STREAM_END)
EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
{ size_t const compBytes = ress->dstBufferSize - strm.avail_out; { size_t const compBytes = ress->dstBufferSize - strm.avail_out;
if (compBytes) { if (compBytes) {
if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
EXM_THROW(73, "Write error : %s", strerror(errno)); EXM_THROW(85, "Write error : %s", strerror(errno));
outFileSize += compBytes; outFileSize += compBytes;
strm.next_out = (BYTE*)ress->dstBuffer; strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize; strm.avail_out = ress->dstBufferSize;
@@ -1493,7 +1502,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
sfnSize = strlen(outDirFilename); sfnSize = strlen(outDirFilename);
assert(outDirFilename != NULL); assert(outDirFilename != NULL);
} }
if (dfnbCapacity <= sfnSize+suffixSize+1) { if (dfnbCapacity <= sfnSize+suffixSize+1) {
/* resize buffer for dstName */ /* resize buffer for dstName */
free(dstFileNameBuffer); free(dstFileNameBuffer);
@@ -1522,9 +1531,10 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
* or into one file each (outFileName == NULL, but suffix != NULL), * or into one file each (outFileName == NULL, but suffix != NULL),
* or into a destination folder (specified with -O) * or into a destination folder (specified with -O)
*/ */
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char* outDirName, unsigned nbFiles, const char** inFileNamesTable, unsigned nbFiles,
const char* outFileName, const char* suffix, const char* outDirName,
const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel, const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams) ZSTD_compressionParameters comprParams)
{ {
@@ -1617,7 +1627,11 @@ static void FIO_freeDResources(dRess_t ress)
/** FIO_fwriteSparse() : /** FIO_fwriteSparse() :
* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ * @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) static unsigned
FIO_fwriteSparse(const FIO_prefs_t* const prefs,
FILE* file,
const void* buffer, size_t bufferSize,
unsigned storedSkips)
{ {
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
size_t bufferSizeT = bufferSize / sizeof(size_t); size_t bufferSizeT = bufferSize / sizeof(size_t);
@@ -1625,6 +1639,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
const size_t* ptrT = bufferT; const size_t* ptrT = bufferT;
static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */
if (prefs->testMode) return 0; /* do not output anything in test mode */
if (!prefs->sparseFileSupport) { /* normal write */ if (!prefs->sparseFileSupport) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
if (sizeCheck != bufferSize) if (sizeCheck != bufferSize)
@@ -1637,7 +1653,7 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (storedSkips > 1 GB) { if (storedSkips > 1 GB) {
int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR); int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
if (seekResult != 0) if (seekResult != 0)
EXM_THROW(71, "1 GB skip error (sparse file support)"); EXM_THROW(91, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB; storedSkips -= 1 GB;
} }
@@ -1653,13 +1669,13 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (nb0T != seg0SizeT) { /* not all 0s */ if (nb0T != seg0SizeT) { /* not all 0s */
int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); if (seekResult) EXM_THROW(92, "Sparse skip error ; try --no-sparse");
storedSkips = 0; storedSkips = 0;
seg0SizeT -= nb0T; seg0SizeT -= nb0T;
ptrT += nb0T; ptrT += nb0T;
{ size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file); { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
if (sizeCheck != seg0SizeT) if (sizeCheck != seg0SizeT)
EXM_THROW(73, "Write error : cannot write decoded block : %s", EXM_THROW(93, "Write error : cannot write decoded block : %s",
strerror(errno)); strerror(errno));
} } } }
ptrT += seg0SizeT; ptrT += seg0SizeT;
@@ -1677,11 +1693,11 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
if (restPtr != restEnd) { if (restPtr != restEnd) {
int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
if (seekResult) if (seekResult)
EXM_THROW(74, "Sparse skip error ; try --no-sparse"); EXM_THROW(94, "Sparse skip error ; try --no-sparse");
storedSkips = 0; storedSkips = 0;
{ size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file); { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
if (sizeCheck != (size_t)(restEnd - restPtr)) if (sizeCheck != (size_t)(restEnd - restPtr))
EXM_THROW(75, "Write error : cannot write decoded end of block : %s", EXM_THROW(95, "Write error : cannot write decoded end of block : %s",
strerror(errno)); strerror(errno));
} } } } } } } }
@@ -1689,8 +1705,9 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
} }
static void static void
FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
{ {
if (prefs->testMode) assert(storedSkips == 0);
if (storedSkips>0) { if (storedSkips>0) {
assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
(void)prefs; /* assert can be disabled, in which case prefs becomes unused */ (void)prefs; /* assert can be disabled, in which case prefs becomes unused */
@@ -1707,7 +1724,7 @@ FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
@return : 0 (no error) */ @return : 0 (no error) */
static int FIO_passThrough(FIO_prefs_t* const prefs, static int FIO_passThrough(const FIO_prefs_t* const prefs,
FILE* foutput, FILE* finput, FILE* foutput, FILE* finput,
void* buffer, size_t bufferSize, void* buffer, size_t bufferSize,
size_t alreadyLoaded) size_t alreadyLoaded)
@@ -1747,7 +1764,10 @@ static unsigned FIO_highbit64(unsigned long long v)
/* FIO_zstdErrorHelp() : /* FIO_zstdErrorHelp() :
* detailed error message when requested window size is too large */ * detailed error message when requested window size is too large */
static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName) static void
FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
const dRess_t* ress,
size_t err, const char* srcFileName)
{ {
ZSTD_frameHeader header; ZSTD_frameHeader header;
@@ -1779,12 +1799,10 @@ static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t er
* @return : size of decoded zstd frame, or an error code * @return : size of decoded zstd frame, or an error code
*/ */
#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
static unsigned long long FIO_decompressZstdFrame( static unsigned long long
FIO_prefs_t* const prefs, FIO_decompressZstdFrame(const FIO_prefs_t* const prefs,
dRess_t* ress, dRess_t* ress, FILE* finput,
FILE* finput, const char* srcFileName, U64 alreadyDecoded)
const char* srcFileName,
U64 alreadyDecoded)
{ {
U64 frameSize = 0; U64 frameSize = 0;
U32 storedSkips = 0; U32 storedSkips = 0;
@@ -1848,13 +1866,16 @@ static unsigned long long FIO_decompressZstdFrame(
#ifdef ZSTD_GZDECOMPRESS #ifdef ZSTD_GZDECOMPRESS
static unsigned long long FIO_decompressGzFrame(dRess_t* ress, static unsigned long long
FILE* srcFile, const char* srcFileName) FIO_decompressGzFrame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* srcFile,
const char* srcFileName)
{ {
unsigned long long outFileSize = 0; unsigned long long outFileSize = 0;
z_stream strm; z_stream strm;
int flush = Z_NO_FLUSH; int flush = Z_NO_FLUSH;
int decodingError = 0; int decodingError = 0;
unsigned storedSkips = 0;
strm.zalloc = Z_NULL; strm.zalloc = Z_NULL;
strm.zfree = Z_NULL; strm.zfree = Z_NULL;
@@ -1889,10 +1910,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
} }
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out; { size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) { if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
decodingError = 1; break;
}
outFileSize += decompBytes; outFileSize += decompBytes;
strm.next_out = (Bytef*)ress->dstBuffer; strm.next_out = (Bytef*)ress->dstBuffer;
strm.avail_out = (uInt)ress->dstBufferSize; strm.avail_out = (uInt)ress->dstBufferSize;
@@ -1909,19 +1927,24 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName);
decodingError = 1; decodingError = 1;
} }
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
} }
#endif #endif
#ifdef ZSTD_LZMADECOMPRESS #ifdef ZSTD_LZMADECOMPRESS
static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma) static unsigned long long
FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* srcFile,
const char* srcFileName, int plain_lzma)
{ {
unsigned long long outFileSize = 0; unsigned long long outFileSize = 0;
lzma_stream strm = LZMA_STREAM_INIT; lzma_stream strm = LZMA_STREAM_INIT;
lzma_action action = LZMA_RUN; lzma_action action = LZMA_RUN;
lzma_ret initRet; lzma_ret initRet;
int decodingError = 0; int decodingError = 0;
unsigned storedSkips = 0;
strm.next_in = 0; strm.next_in = 0;
strm.avail_in = 0; strm.avail_in = 0;
@@ -1964,10 +1987,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
} }
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out; { size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) { if (decompBytes) {
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
decodingError = 1; break;
}
outFileSize += decompBytes; outFileSize += decompBytes;
strm.next_out = (BYTE*)ress->dstBuffer; strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize; strm.avail_out = ress->dstBufferSize;
@@ -1979,19 +1999,23 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
memmove(ress->srcBuffer, strm.next_in, strm.avail_in); memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
ress->srcBufferLoaded = strm.avail_in; ress->srcBufferLoaded = strm.avail_in;
lzma_end(&strm); lzma_end(&strm);
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
} }
#endif #endif
#ifdef ZSTD_LZ4DECOMPRESS #ifdef ZSTD_LZ4DECOMPRESS
static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, static unsigned long long
FILE* srcFile, const char* srcFileName) FIO_decompressLz4Frame(const FIO_prefs_t* const prefs,
dRess_t* ress, FILE* srcFile,
const char* srcFileName)
{ {
unsigned long long filesize = 0; unsigned long long filesize = 0;
LZ4F_errorCode_t nextToLoad; LZ4F_errorCode_t nextToLoad;
LZ4F_decompressionContext_t dCtx; LZ4F_decompressionContext_t dCtx;
LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
int decodingError = 0; int decodingError = 0;
unsigned storedSkips = 0;
if (LZ4F_isError(errorCode)) { if (LZ4F_isError(errorCode)) {
DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n");
@@ -2035,10 +2059,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
/* Write Block */ /* Write Block */
if (decodedBytes) { if (decodedBytes) {
if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) { storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decodedBytes, storedSkips);
DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
decodingError = 1; nextToLoad = 0; break;
}
filesize += decodedBytes; filesize += decodedBytes;
DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20));
} }
@@ -2059,6 +2080,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
LZ4F_freeDecompressionContext(dCtx); LZ4F_freeDecompressionContext(dCtx);
ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */
FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; return decodingError ? FIO_ERROR_FRAME_DECODING : filesize;
} }
@@ -2072,8 +2094,9 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
* @return : 0 : OK * @return : 0 : OK
* 1 : error * 1 : error
*/ */
static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName) dRess_t ress, FILE* srcFile,
const char* dstFileName, const char* srcFileName)
{ {
unsigned readSomething = 0; unsigned readSomething = 0;
unsigned long long filesize = 0; unsigned long long filesize = 0;
@@ -2105,7 +2128,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
filesize += frameSize; filesize += frameSize;
} else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
#ifdef ZSTD_GZDECOMPRESS #ifdef ZSTD_GZDECOMPRESS
unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName); unsigned long long const frameSize = FIO_decompressGzFrame(prefs, &ress, srcFile, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize; filesize += frameSize;
#else #else
@@ -2115,7 +2138,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
} else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */
|| (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
#ifdef ZSTD_LZMADECOMPRESS #ifdef ZSTD_LZMADECOMPRESS
unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); unsigned long long const frameSize = FIO_decompressLzmaFrame(prefs, &ress, srcFile, srcFileName, buf[0] != 0xFD);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize; filesize += frameSize;
#else #else
@@ -2124,7 +2147,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
#endif #endif
} else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
#ifdef ZSTD_LZ4DECOMPRESS #ifdef ZSTD_LZ4DECOMPRESS
unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName); unsigned long long const frameSize = FIO_decompressLz4Frame(prefs, &ress, srcFile, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize; filesize += frameSize;
#else #else
@@ -2164,11 +2187,11 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
int transfer_permissions = 0; int transfer_permissions = 0;
int releaseDstFile = 0; int releaseDstFile = 0;
if (ress.dstFile == NULL) { if ((ress.dstFile == NULL) && (prefs->testMode==0)) {
releaseDstFile = 1; releaseDstFile = 1;
ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName); ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
if (ress.dstFile==0) return 1; if (ress.dstFile==NULL) return 1;
/* Must only be added after FIO_openDstFile() succeeds. /* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists, * Otherwise we may delete the destination file if it already exists,
@@ -2181,7 +2204,6 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
transfer_permissions = 1; transfer_permissions = 1;
} }
result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName); result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName);
if (releaseDstFile) { if (releaseDstFile) {
@@ -2278,7 +2300,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
char* outDirFilename = NULL; char* outDirFilename = NULL;
size_t sfnSize = strlen(srcFileName); size_t sfnSize = strlen(srcFileName);
size_t suffixSize; size_t suffixSize;
const char* const suffixPtr = strrchr(srcFileName, '.'); const char* const suffixPtr = strrchr(srcFileName, '.');
if (suffixPtr == NULL) { if (suffixPtr == NULL) {
DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
@@ -2328,7 +2350,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
dfnbCapacity = sfnSize + 20; dfnbCapacity = sfnSize + 20;
dstFileNameBuffer = (char*)malloc(dfnbCapacity); dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (dstFileNameBuffer==NULL) if (dstFileNameBuffer==NULL)
EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno)); EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
} }
/* return dst name == src name truncated from suffix */ /* return dst name == src name truncated from suffix */
@@ -2357,11 +2379,13 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
if (outFileName) { if (outFileName) {
unsigned u; unsigned u;
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (!prefs->testMode) {
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName);
}
for (u=0; u<nbFiles; u++) for (u=0; u<nbFiles; u++)
error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]); error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]);
if (fclose(ress.dstFile)) if ((!prefs->testMode) && (fclose(ress.dstFile)))
EXM_THROW(72, "Write error : %s : cannot properly close output file", EXM_THROW(72, "Write error : %s : cannot properly close output file",
strerror(errno)); strerror(errno));
} else { } else {

24
programs/fileio.h
View File

@@ -26,7 +26,7 @@ extern "C" {
#define stdinmark "/*stdin*\\" #define stdinmark "/*stdin*\\"
#define stdoutmark "/*stdout*\\" #define stdoutmark "/*stdout*\\"
#ifdef _WIN32 #ifdef _WIN32
# define nulmark "nul" # define nulmark "NUL"
#else #else
# define nulmark "/dev/null" # define nulmark "/dev/null"
#endif #endif
@@ -74,6 +74,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize); void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize); void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint); void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint);
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode);
void FIO_setLiteralCompressionMode( void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs, FIO_prefs_t* const prefs,
ZSTD_literalCompressionMode_e mode); ZSTD_literalCompressionMode_e mode);
@@ -85,14 +86,14 @@ void FIO_setNotificationLevel(int level);
* Single File functions * Single File functions
***************************************/ ***************************************/
/** FIO_compressFilename() : /** FIO_compressFilename() :
@return : 0 == ok; 1 == pb with src file. */ * @return : 0 == ok; 1 == pb with src file. */
int FIO_compressFilename (FIO_prefs_t* const prefs, int FIO_compressFilename (FIO_prefs_t* const prefs,
const char* outfilename, const char* infilename, const char* outfilename, const char* infilename,
const char* dictFileName, int compressionLevel, const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams); ZSTD_compressionParameters comprParams);
/** FIO_decompressFilename() : /** FIO_decompressFilename() :
@return : 0 == ok; 1 == pb with src file. */ * @return : 0 == ok; 1 == pb with src file. */
int FIO_decompressFilename (FIO_prefs_t* const prefs, int FIO_decompressFilename (FIO_prefs_t* const prefs,
const char* outfilename, const char* infilename, const char* dictFileName); const char* outfilename, const char* infilename, const char* dictFileName);
@@ -103,15 +104,16 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
* Multiple File functions * Multiple File functions
***************************************/ ***************************************/
/** FIO_compressMultipleFilenames() : /** FIO_compressMultipleFilenames() :
@return : nb of missing files */ * @return : nb of missing files */
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char* outDirName, unsigned nbFiles, const char** inFileNamesTable, unsigned nbFiles,
const char* outFileName, const char* suffix, const char* outDirName,
const char* dictFileName, int compressionLevel, const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams); ZSTD_compressionParameters comprParams);
/** FIO_decompressMultipleFilenames() : /** FIO_decompressMultipleFilenames() :
@return : nb of missing or skipped files */ * @return : nb of missing or skipped files */
int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles, const char** srcNamesTable, unsigned nbFiles,
const char* outDirName, const char* outDirName,
@@ -119,10 +121,12 @@ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char* dictFileName); const char* dictFileName);
/* FIO_checkFilenameCollisions() : /* FIO_checkFilenameCollisions() :
* Checks for and warns if thereå are any files that would have the same output path * Checks for and warns if there are any files that would have the same output path
*/ */
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles); int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
/*-************************************* /*-*************************************
* Advanced stuff (should actually be hosted elsewhere) * Advanced stuff (should actually be hosted elsewhere)
***************************************/ ***************************************/

72
programs/util.c
View File

@@ -106,20 +106,23 @@ int UTIL_compareStr(const void *p1, const void *p2) {
return strcmp(* (char * const *) p1, * (char * const *) p2); return strcmp(* (char * const *) p1, * (char * const *) p2);
} }
int UTIL_isSameFile(const char* file1, const char* file2) int UTIL_isSameFile(const char* fName1, const char* fName2)
{ {
#if defined(_MSC_VER) assert(fName1 != NULL); assert(fName2 != NULL);
#if defined(_MSC_VER) || defined(_WIN32)
/* note : Visual does not support file identification by inode. /* note : Visual does not support file identification by inode.
* inode does not work on Windows, even with a posix layer, like msys2.
* The following work-around is limited to detecting exact name repetition only, * The following work-around is limited to detecting exact name repetition only,
* aka `filename` is considered different from `subdir/../filename` */ * aka `filename` is considered different from `subdir/../filename` */
return !strcmp(file1, file2); return !strcmp(fName1, fName2);
#else #else
stat_t file1Stat; { stat_t file1Stat;
stat_t file2Stat; stat_t file2Stat;
return UTIL_getFileStat(file1, &file1Stat) return UTIL_getFileStat(fName1, &file1Stat)
&& UTIL_getFileStat(file2, &file2Stat) && UTIL_getFileStat(fName2, &file2Stat)
&& (file1Stat.st_dev == file2Stat.st_dev) && (file1Stat.st_dev == file2Stat.st_dev)
&& (file1Stat.st_ino == file2Stat.st_ino); && (file1Stat.st_ino == file2Stat.st_ino);
}
#endif #endif
} }
@@ -240,19 +243,20 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
DIR *dir; DIR *dir;
struct dirent *entry; struct dirent *entry;
char* path; char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0; size_t dirLength, fnameLength, pathLength;
int nbFiles = 0;
if (!(dir = opendir(dirName))) { if (!(dir = opendir(dirName))) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0; return 0;
} }
dirLength = (int)strlen(dirName); dirLength = strlen(dirName);
errno = 0; errno = 0;
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
if (strcmp (entry->d_name, "..") == 0 || if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue; strcmp (entry->d_name, ".") == 0) continue;
fnameLength = (int)strlen(entry->d_name); fnameLength = strlen(entry->d_name);
path = (char*) malloc(dirLength + fnameLength + 2); path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { closedir(dir); return 0; } if (!path) { closedir(dir); return 0; }
memcpy(path, dirName, dirLength); memcpy(path, dirName, dirLength);
@@ -274,7 +278,8 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
} else { } else {
if (*bufStart + *pos + pathLength >= *bufEnd) { if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize); assert(newListSize >= 0);
*bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
*bufEnd = *bufStart + newListSize; *bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; } if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
} }
@@ -323,7 +328,6 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
unsigned i, nbFiles; unsigned i, nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE); char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE; char* bufend = buf + LIST_SIZE_INCREASE;
const char** fileTable;
if (!buf) return NULL; if (!buf) return NULL;
@@ -332,36 +336,37 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
size_t const len = strlen(inputNames[i]); size_t const len = strlen(inputNames[i]);
if (buf + pos + len >= bufend) { if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
buf = (char*)UTIL_realloc(buf, newListSize); assert(newListSize >= 0);
buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
bufend = buf + newListSize; bufend = buf + newListSize;
if (!buf) return NULL; if (!buf) return NULL;
} }
if (buf + pos + len < bufend) { if (buf + pos + len < bufend) {
memcpy(buf+pos, inputNames[i], len+1); /* with final \0 */ memcpy(buf+pos, inputNames[i], len+1); /* including final \0 */
pos += len + 1; pos += len + 1;
nbFiles++; nbFiles++;
} }
} else { } else {
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks); nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
if (buf == NULL) return NULL; if (buf == NULL) return NULL;
} } } }
if (nbFiles == 0) { free(buf); return NULL; } if (nbFiles == 0) { free(buf); return NULL; }
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); { const char** const fileTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileTable));
if (!fileTable) { free(buf); return NULL; } if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++) { for (i = 0, pos = 0; i < nbFiles; i++) {
fileTable[i] = buf + pos; fileTable[i] = buf + pos;
pos += strlen(fileTable[i]) + 1; if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
pos += strlen(fileTable[i]) + 1;
}
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
return fileTable;
} }
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
return fileTable;
} }
@@ -394,8 +399,13 @@ int UTIL_countPhysicalCores(void)
DWORD returnLength = 0; DWORD returnLength = 0;
size_t byteOffset = 0; size_t byteOffset = 0;
glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")), #if defined(_MSC_VER)
"GetLogicalProcessorInformation"); /* Visual Studio does not like the following cast */
# pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */
# pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */
#endif
glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
"GetLogicalProcessorInformation");
if (glpi == NULL) { if (glpi == NULL) {
goto failed; goto failed;

BIN
programs/windres/zstd32.res
View File

Binary file not shown.

BIN
programs/windres/zstd64.res
View File

Binary file not shown.

56
programs/zstdcli.c
View File

@@ -287,7 +287,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
* @return 0 and doesn't modify *stringPtr otherwise. * @return 0 and doesn't modify *stringPtr otherwise.
*/ */
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) static int longCommandWArg(const char** stringPtr, const char* longCommand)
{ {
size_t const comSize = strlen(longCommand); size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize); int const result = !strncmp(*stringPtr, longCommand, comSize);
@@ -430,8 +430,8 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void)
static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr) static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
{ {
for ( ; ;) { for ( ; ;) {
if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
DISPLAYLEVEL(4, "invalid compression parameter \n"); DISPLAYLEVEL(4, "invalid compression parameter \n");
return 0; return 0;
} }
@@ -526,7 +526,7 @@ static int init_cLevel(void) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env); DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
return ZSTDCLI_CLEVEL_DEFAULT; return ZSTDCLI_CLEVEL_DEFAULT;
} else if (*ptr == 0) { } else if (*ptr == 0) {
return sign * absLevel; return sign * (int)absLevel;
} }
} }
@@ -584,7 +584,7 @@ int main(int argCount, const char* argv[])
int cLevelLast = -1000000000; int cLevelLast = -1000000000;
unsigned recursive = 0; unsigned recursive = 0;
unsigned memLimit = 0; unsigned memLimit = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ const char** filenameTable = (const char**)malloc((size_t)argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0; unsigned filenameIdx = 0;
const char* programName = argv[0]; const char* programName = argv[0];
const char* outFileName = NULL; const char* outFileName = NULL;
@@ -745,7 +745,7 @@ int main(int argCount, const char* argv[])
continue; continue;
} }
#endif #endif
if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
@@ -807,7 +807,7 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
/* compression Level */ /* compression Level */
if ((*argument>='0') && (*argument<='9')) { if ((*argument>='0') && (*argument<='9')) {
dictCLevel = cLevel = readU32FromChar(&argument); dictCLevel = cLevel = (int)readU32FromChar(&argument);
continue; continue;
} }
#endif #endif
@@ -856,7 +856,7 @@ int main(int argCount, const char* argv[])
/* destination file name */ /* destination file name */
case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
/* limit decompression memory */ /* limit decompression memory */
case 'M': case 'M':
argument++; argument++;
@@ -879,7 +879,7 @@ int main(int argCount, const char* argv[])
case 'e': case 'e':
/* compression Level */ /* compression Level */
argument++; argument++;
cLevelLast = readU32FromChar(&argument); cLevelLast = (int)readU32FromChar(&argument);
break; break;
/* Modify Nb Iterations (benchmark only) */ /* Modify Nb Iterations (benchmark only) */
@@ -905,7 +905,7 @@ int main(int argCount, const char* argv[])
/* nb of threads (hidden option) */ /* nb of threads (hidden option) */
case 'T': case 'T':
argument++; argument++;
nbWorkers = readU32FromChar(&argument); nbWorkers = (int)readU32FromChar(&argument);
break; break;
/* Dictionary Selection level */ /* Dictionary Selection level */
@@ -1042,16 +1042,16 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
benchParams.blockSize = blockSize; benchParams.blockSize = blockSize;
benchParams.nbWorkers = nbWorkers; benchParams.nbWorkers = nbWorkers;
benchParams.realTime = setRealTimePrio; benchParams.realTime = (unsigned)setRealTimePrio;
benchParams.nbSeconds = bench_nbSeconds; benchParams.nbSeconds = bench_nbSeconds;
benchParams.ldmFlag = ldmFlag; benchParams.ldmFlag = ldmFlag;
benchParams.ldmMinMatch = g_ldmMinMatch; benchParams.ldmMinMatch = (int)g_ldmMinMatch;
benchParams.ldmHashLog = g_ldmHashLog; benchParams.ldmHashLog = (int)g_ldmHashLog;
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog; benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
} }
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) { if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
benchParams.ldmHashRateLog = g_ldmHashRateLog; benchParams.ldmHashRateLog = (int)g_ldmHashRateLog;
} }
benchParams.literalCompressionMode = literalCompressionMode; benchParams.literalCompressionMode = literalCompressionMode;
@@ -1092,16 +1092,16 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NODICT #ifndef ZSTD_NODICT
ZDICT_params_t zParams; ZDICT_params_t zParams;
zParams.compressionLevel = dictCLevel; zParams.compressionLevel = dictCLevel;
zParams.notificationLevel = g_displayLevel; zParams.notificationLevel = (unsigned)g_displayLevel;
zParams.dictID = dictID; zParams.dictID = dictID;
if (dict == cover) { if (dict == cover) {
int const optimize = !coverParams.k || !coverParams.d; int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = nbWorkers; coverParams.nbThreads = (unsigned)nbWorkers;
coverParams.zParams = zParams; coverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize); operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
} else if (dict == fastCover) { } else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d; int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = nbWorkers; fastCoverParams.nbThreads = (unsigned)nbWorkers;
fastCoverParams.zParams = zParams; fastCoverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize); operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
} else { } else {
@@ -1120,7 +1120,7 @@ int main(int argCount, const char* argv[])
} }
#ifndef ZSTD_NODECOMPRESS #ifndef ZSTD_NODECOMPRESS
if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */ if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
#endif #endif
/* No input filename ==> use stdin and stdout */ /* No input filename ==> use stdin and stdout */
@@ -1156,14 +1156,14 @@ int main(int argCount, const char* argv[])
if (operation==zom_compress) { if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
FIO_setNbWorkers(prefs, nbWorkers); FIO_setNbWorkers(prefs, nbWorkers);
FIO_setBlockSize(prefs, (U32)blockSize); FIO_setBlockSize(prefs, (int)blockSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog); if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
FIO_setLdmFlag(prefs, ldmFlag); FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
FIO_setLdmHashLog(prefs, g_ldmHashLog); FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
FIO_setLdmMinMatch(prefs, g_ldmMinMatch); FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog); if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog); if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
FIO_setAdaptiveMode(prefs, adapt); FIO_setAdaptiveMode(prefs, (unsigned)adapt);
FIO_setAdaptMin(prefs, adaptMin); FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(prefs, adaptMax); FIO_setAdaptMax(prefs, adaptMax);
FIO_setRsyncable(prefs, rsyncable); FIO_setRsyncable(prefs, rsyncable);
@@ -1177,7 +1177,7 @@ int main(int argCount, const char* argv[])
if ((filenameIdx==1) && outFileName) if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
else else
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else #else
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n"); DISPLAY("Compression not supported \n");

7
tests/Makefile
View File

@@ -250,7 +250,8 @@ clean:
$(MAKE) -C $(ZSTDDIR) clean $(MAKE) -C $(ZSTDDIR) clean
$(MAKE) -C $(PRGDIR) clean $(MAKE) -C $(PRGDIR) clean
@$(RM) -fR $(TESTARTEFACT) @$(RM) -fR $(TESTARTEFACT)
@$(RM) -f core *.o tmp* *.tmp result* *.gcda dictionary *.zst \ @$(RM) -rf tmp* # some test directories are named tmp*
@$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \
fullbench-lib$(EXT) fullbench-dll$(EXT) \ fullbench-lib$(EXT) fullbench-dll$(EXT) \
@@ -264,7 +265,7 @@ clean:
#---------------------------------------------------------------------------------- #----------------------------------------------------------------------------------
#make valgrindTest is validated only for Linux, macOS, BSD, Hurd and Solaris targets # valgrind tests are validated only for some posix platforms
#---------------------------------------------------------------------------------- #----------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
HOST_OS = POSIX HOST_OS = POSIX
@@ -286,7 +287,7 @@ valgrindTest: zstd datagen fuzzer fullbench
endif endif
ifneq (,$(filter MSYS%,$(shell uname))) ifneq (,$(filter MINGW% MSYS%,$(shell uname)))
HOST_OS = MSYS HOST_OS = MSYS
endif endif

42
tests/playTests.sh
View File

@@ -64,11 +64,12 @@ PRGDIR="$SCRIPT_DIR/../programs"
TESTDIR="$SCRIPT_DIR/../tests" TESTDIR="$SCRIPT_DIR/../tests"
UNAME=$(uname) UNAME=$(uname)
isTerminal=false detectedTerminal=false
if [ -t 0 ] && [ -t 1 ] if [ -t 0 ] && [ -t 1 ]
then then
isTerminal=true detectedTerminal=true
fi fi
isTerminal=${isTerminal:-$detectedTerminal}
isWindows=false isWindows=false
INTOVOID="/dev/null" INTOVOID="/dev/null"
@@ -211,8 +212,8 @@ $ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t
$ZSTD tmp -c --compress-literals -1 | $ZSTD -t $ZSTD tmp -c --compress-literals -1 | $ZSTD -t
$ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t $ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t
$ZSTD tmp -c --compress-literals -19 | $ZSTD -t $ZSTD tmp -c --compress-literals -19 | $ZSTD -t
$ZSTD -b --fast=1 -i1e1 tmp --compress-literals $ZSTD -b --fast=1 -i0e1 tmp --compress-literals
$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals $ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
println "test : file removal" println "test : file removal"
$ZSTD -f --rm tmp $ZSTD -f --rm tmp
@@ -239,7 +240,7 @@ rm tmp # erase source file
touch tmp.zst # create destination file touch tmp.zst # create destination file
$ZSTD -f tmp && die "attempt to compress a non existing file" $ZSTD -f tmp && die "attempt to compress a non existing file"
test -f tmp.zst # destination file should still be present test -f tmp.zst # destination file should still be present
rm tmp* rm -rf tmp* # may also erase tmp* directory from previous failed run
println "\n===> decompression only tests " println "\n===> decompression only tests "
head -c 1048576 /dev/zero > tmp head -c 1048576 /dev/zero > tmp
@@ -283,7 +284,7 @@ test -f tmpOutDir/tmp1.zst
test -f tmpOutDir/tmp2.zst test -f tmpOutDir/tmp2.zst
println "test : decompress multiple files into an output directory, --output-dir-flat" println "test : decompress multiple files into an output directory, --output-dir-flat"
mkdir tmpOutDirDecomp mkdir tmpOutDirDecomp
$ZSTD tmpOutDir/ -r -d --output-dir-flat tmpOutDirDecomp $ZSTD tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2 test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1 test -f tmpOutDirDecomp/tmp1
rm -rf tmp* rm -rf tmp*
@@ -577,30 +578,27 @@ println "- Create dictionary with short dictID"
$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 $ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionaries with shrink-dict flag enabled" println "- Create dictionaries with shrink-dict flag enabled"
$ZSTD --train-fastcover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict $ZSTD --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
$ZSTD --train-fastcover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 $ZSTD --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
$ZSTD --train-fastcover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 $ZSTD --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
println "- Create dictionary with size limit" println "- Create dictionary with size limit"
$ZSTD --train-fastcover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K $ZSTD --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Compare size of dictionary from 90% training samples with 80% training samples"
$ZSTD --train-fastcover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using all samples for both training and testing" println "- Create dictionary using all samples for both training and testing"
$ZSTD --train-fastcover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using f=16" println "- Create dictionary using f=16"
$ZSTD --train-fastcover=f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
$ZSTD --train-fastcover=accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" $ZSTD --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
println "- Create dictionary using accel=2" println "- Create dictionary using accel=2"
$ZSTD --train-fastcover=accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using accel=10" println "- Create dictionary using accel=10"
$ZSTD --train-fastcover=accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary with multithreading" println "- Create dictionary with multithreading"
$ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Test -o before --train-fastcover" println "- Test -o before --train-fastcover"
rm -f tmpDict dictionary rm -f tmpDict dictionary
$ZSTD -o tmpDict --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict test -f tmpDict
$ZSTD --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c $ZSTD --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary test -f dictionary
rm tmp* dictionary rm tmp* dictionary
@@ -674,10 +672,10 @@ $ZSTD -i0b0e3 tmp1
println "bench negative level" println "bench negative level"
$ZSTD -bi0 --fast tmp1 $ZSTD -bi0 --fast tmp1
println "with recursive and quiet modes" println "with recursive and quiet modes"
$ZSTD -rqi1b1e2 tmp1 $ZSTD -rqi0b1e2 tmp1
println "benchmark decompression only" println "benchmark decompression only"
$ZSTD -f tmp1 $ZSTD -f tmp1
$ZSTD -b -d -i1 tmp1.zst $ZSTD -b -d -i0 tmp1.zst
println "\n===> zstd compatibility tests " println "\n===> zstd compatibility tests "