diff --git a/.travis.yml b/.travis.yml index 20fa9beeb..a6e1a99ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,10 @@ matrix: - name: Trusty (clang-3.8 + MSan + Test Zstd) script: - make clang38install - - CC=clang-3.8 make clean msan-test-zstd + # External libraries must be turned off when using MSAN tests, + # because they are not msan-instrumented, + # so any data coming from these libraries is always considered "uninitialized" + - CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 - name: Trusty (Minimal Decompressor Macros) script: diff --git a/appveyor.yml b/appveyor.yml index 35f019dd6..87fa5c12d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,3 +1,7 @@ +# Following tests are run _only_ on master branch +# To reproduce these tests, it's possible to push into a branch `appveyorTest` +# or a branch `visual*`, they will intentionnally trigger `master` tests + - version: 1.0.{build} branches: @@ -176,6 +180,11 @@ fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% ) + +# The following tests are for regular pushes +# into `dev` or some feature branch +# There run less tests, for shorter feedback loop + - version: 1.0.{build} environment: @@ -249,3 +258,11 @@ COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ ) + + + test_script: + - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% + - if [%HOST%]==[mingw] ( + set "CC=%COMPILER%" && + make check + ) diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile index b2ed9f33d..704f86766 100644 --- a/doc/educational_decoder/Makefile +++ b/doc/educational_decoder/Makefile @@ -7,8 +7,15 @@ # in the COPYING file in the root directory of this source tree). # ################################################################ -ZSTD ?= zstd # requires zstd installation on local system +ZSTD ?= zstd # note: requires zstd installation on local system + +UNAME?= $(shell uname) +ifeq ($(UNAME), SunOS) +DIFF ?= gdiff +else DIFF ?= diff +endif + HARNESS_FILES=*.c MULTITHREAD_LDFLAGS = -pthread @@ -29,26 +36,27 @@ harness: $(HARNESS_FILES) $(CC) $(FLAGS) $^ -o $@ clean: - @$(RM) -f harness - @$(RM) -rf harness.dSYM + @$(RM) harness + @$(RM) -rf harness.dSYM # MacOS specific test: harness # # Testing single-file decompression with educational decoder # - @$(ZSTD) README.md -o tmp.zst + @$(ZSTD) -f README.md -o tmp.zst @./harness tmp.zst tmp @$(DIFF) -s tmp README.md - @$(RM) -f tmp* + @$(RM) tmp* # # Testing dictionary decompression with education decoder # # note : files are presented multiple for training, to reach minimum threshold @$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \ harness.c zstd_decompress.c zstd_decompress.h README.md \ - harness.c zstd_decompress.c zstd_decompress.h README.md - @$(ZSTD) -D dictionary README.md -o tmp.zst + harness.c zstd_decompress.c zstd_decompress.h README.md \ + -o dictionary + @$(ZSTD) -f README.md -D dictionary -o tmp.zst @./harness tmp.zst tmp dictionary @$(DIFF) -s tmp README.md - @$(RM) -f tmp* dictionary + @$(RM) tmp* dictionary @$(MAKE) clean diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c index 36f3967a9..a704f6bdb 100644 --- a/doc/educational_decoder/harness.c +++ b/doc/educational_decoder/harness.c @@ -21,88 +21,90 @@ typedef unsigned char u8; // Protect against allocating too much memory for output #define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024) -u8 *input; -u8 *output; -u8 *dict; - -size_t read_file(const char *path, u8 **ptr) { - FILE *f = fopen(path, "rb"); +static size_t read_file(const char *path, u8 **ptr) +{ + FILE* const f = fopen(path, "rb"); if (!f) { - fprintf(stderr, "failed to open file %s\n", path); + fprintf(stderr, "failed to open file %s \n", path); exit(1); } fseek(f, 0L, SEEK_END); - size_t size = (size_t)ftell(f); + size_t const size = (size_t)ftell(f); rewind(f); *ptr = malloc(size); if (!ptr) { - fprintf(stderr, "failed to allocate memory to hold %s\n", path); + fprintf(stderr, "failed to allocate memory to hold %s \n", path); exit(1); } - size_t pos = 0; - while (!feof(f)) { - size_t read = fread(&(*ptr)[pos], 1, size, f); - if (ferror(f)) { - fprintf(stderr, "error while reading file %s\n", path); - exit(1); - } - pos += read; + size_t const read = fread(*ptr, 1, size, f); + if (read != size) { /* must read everything in one pass */ + fprintf(stderr, "error while reading file %s \n", path); + exit(1); } fclose(f); - return pos; + return read; } -void write_file(const char *path, const u8 *ptr, size_t size) { - FILE *f = fopen(path, "wb"); +static void write_file(const char *path, const u8 *ptr, size_t size) +{ + FILE* const f = fopen(path, "wb"); + if (!f) { + fprintf(stderr, "failed to open file %s \n", path); + exit(1); + } size_t written = 0; while (written < size) { - written += fwrite(&ptr[written], 1, size, f); + written += fwrite(ptr+written, 1, size, f); if (ferror(f)) { fprintf(stderr, "error while writing file %s\n", path); exit(1); - } - } + } } fclose(f); } -int main(int argc, char **argv) { +int main(int argc, char **argv) +{ if (argc < 3) { - fprintf(stderr, "usage: %s [dictionary]\n", + fprintf(stderr, "usage: %s [dictionary] \n", argv[0]); return 1; } - size_t input_size = read_file(argv[1], &input); + u8* input; + size_t const input_size = read_file(argv[1], &input); + + u8* dict = NULL; size_t dict_size = 0; if (argc >= 4) { dict_size = read_file(argv[3], &dict); } - size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size); - if (decompressed_size == (size_t)-1) { - decompressed_size = MAX_COMPRESSION_RATIO * input_size; + size_t out_capacity = ZSTD_get_decompressed_size(input, input_size); + if (out_capacity == (size_t)-1) { + out_capacity = MAX_COMPRESSION_RATIO * input_size; fprintf(stderr, "WARNING: Compressed data does not contain " "decompressed size, going to assume the compression " "ratio is at most %d (decompressed size of at most " - "%zu)\n", - MAX_COMPRESSION_RATIO, decompressed_size); + "%u) \n", + MAX_COMPRESSION_RATIO, (unsigned)out_capacity); } - if (decompressed_size > MAX_OUTPUT_SIZE) { + if (out_capacity > MAX_OUTPUT_SIZE) { fprintf(stderr, - "Required output size too large for this implementation\n"); + "Required output size too large for this implementation \n"); return 1; } - output = malloc(decompressed_size); + + u8* const output = malloc(out_capacity); if (!output) { - fprintf(stderr, "failed to allocate memory\n"); + fprintf(stderr, "failed to allocate memory \n"); return 1; } @@ -110,16 +112,17 @@ int main(int argc, char **argv) { if (dict) { parse_dictionary(parsed_dict, dict, dict_size); } - size_t decompressed = - ZSTD_decompress_with_dict(output, decompressed_size, - input, input_size, parsed_dict); + size_t const decompressed_size = + ZSTD_decompress_with_dict(output, out_capacity, + input, input_size, + parsed_dict); free_dictionary(parsed_dict); - write_file(argv[2], output, decompressed); + write_file(argv[2], output, decompressed_size); free(input); free(output); free(dict); - input = output = dict = NULL; + return 0; } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 18492acb4..fc9765bd3 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -490,7 +490,7 @@ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { } MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { - return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace; + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); } MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { diff --git a/programs/fileio.c b/programs/fileio.c index eecdf0dd3..f4384484c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -284,9 +284,10 @@ void FIO_addAbortHandler() /*-************************************* -* Parameters: Typedefs +* Parameters: FIO_prefs_t ***************************************/ +/* typedef'd to FIO_prefs_t within fileio.h */ struct FIO_prefs_s { /* Algorithm preferences */ @@ -308,6 +309,7 @@ struct FIO_prefs_s { size_t streamSrcSize; size_t targetCBlockSize; int srcSizeHint; + int testMode; ZSTD_literalCompressionMode_e literalCompressionMode; /* IO preferences */ @@ -355,6 +357,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->streamSrcSize = 0; ret->targetCBlockSize = 0; ret->srcSizeHint = 0; + ret->testMode = 0; ret->literalCompressionMode = ZSTD_lcm_auto; return ret; } @@ -435,6 +438,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) { prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); } +void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) { + prefs->testMode = (testMode!=0); +} + void FIO_setLiteralCompressionMode( FIO_prefs_t* const prefs, ZSTD_literalCompressionMode_e mode) { @@ -529,8 +536,12 @@ static FILE* FIO_openSrcFile(const char* srcFileName) /** FIO_openDstFile() : * condition : `dstFileName` must be non-NULL. * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName) +static FILE* +FIO_openDstFile(FIO_prefs_t* const prefs, + const char* srcFileName, const char* dstFileName) { + if (prefs->testMode) return NULL; /* do not open file in test mode */ + assert(dstFileName != NULL); if (!strcmp (dstFileName, stdoutmark)) { DISPLAYLEVEL(4,"Using stdout for output \n"); @@ -555,10 +566,14 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, if (UTIL_isRegularFile(dstFileName)) { /* Check if destination file already exists */ FILE* const fCheck = fopen( dstFileName, "rb" ); +#if !defined(_WIN32) + /* this test does not work on Windows : + * `NUL` and `nul` are detected as regular files */ if (!strcmp(dstFileName, nulmark)) { EXM_THROW(40, "%s is unexpectedly categorized as a regular file", dstFileName); } +#endif if (fCheck != NULL) { /* dst file exists, authorization prompt */ fclose(fCheck); if (!prefs->overwrite) { @@ -648,7 +663,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { DISPLAY("Unable to malloc new str array, not checking for name collisions\n"); return 1; } - + for (u = 0; u < nbFiles; ++u) { filename = strrchr(filenameTable[u], c[0]); if (filename == NULL) { @@ -671,56 +686,50 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { return 0; } +static const char* +extractFilename(const char* path, char separator) +{ + const char* search = strrchr(path, separator); + if (search == NULL) return path; + return search+1; +} + /* FIO_createFilename_fromOutDir() : * Takes a source file name and specified output directory, and * allocates memory for and returns a pointer to final path. * This function never returns an error (it may abort() in case of pb) */ static char* -FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, const size_t suffixLen) +FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen) { - const char* c, *filenameBegin; - char* filename, *result; - size_t finalPathLen; + const char* filenameStart; + char separator; + char* result; - #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ - c = "\\"; - #else - c = "/"; - #endif +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + separator = '\\'; +#else + separator = '/'; +#endif - finalPathLen = strlen(outDirName); - filenameBegin = strrchr(srcFilename, c[0]); - if (filenameBegin == NULL) { - filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char)); - if (!filename) { - EXM_THROW(30, "zstd: %s", strerror(errno)); - } - strcpy(filename, srcFilename); - } else { - filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char)); - if (!filename) { - EXM_THROW(30, "zstd: %s", strerror(errno)); - } - strcpy(filename, filenameBegin+1); - } + filenameStart = extractFilename(path, separator); +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + filenameStart = extractFilename(filenameStart, '/'); /* sometimes, '/' separator is also used on Windows (mingw+msys2) */ +#endif - finalPathLen += strlen(filename); - result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char)); + result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1); if (!result) { - free(filename); - EXM_THROW(30, "zstd: %s", strerror(errno)); + EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno)); } - strcpy(result, outDirName); - if (outDirName[strlen(outDirName)-1] == c[0]) { - strcat(result, filename); - } else { - strcat(result, c); - strcat(result, filename); + memcpy(result, outDirName, strlen(outDirName)); + if (outDirName[strlen(outDirName)-1] == separator) { + memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart)); + } else { + memcpy(result + strlen(outDirName), &separator, 1); + memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart)); } - free(filename); return result; } @@ -825,13 +834,12 @@ static void FIO_freeCResources(cRess_t ress) #ifdef ZSTD_GZCOMPRESS static unsigned long long -FIO_compressGzFrame(cRess_t* ress, +FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */ const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize) { unsigned long long inFileSize = 0, outFileSize = 0; z_stream strm; - int ret; if (compressionLevel > Z_BEST_COMPRESSION) compressionLevel = Z_BEST_COMPRESSION; @@ -840,11 +848,12 @@ FIO_compressGzFrame(cRess_t* ress, strm.zfree = Z_NULL; strm.opaque = Z_NULL; - ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, + { int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ - if (ret != Z_OK) - EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); + if (ret != Z_OK) { + EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); + } } strm.next_in = 0; strm.avail_in = 0; @@ -852,6 +861,7 @@ FIO_compressGzFrame(cRess_t* ress, strm.avail_out = (uInt)ress->dstBufferSize; while (1) { + int ret; if (strm.avail_in == 0) { size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); if (inSize == 0) break; @@ -862,32 +872,31 @@ FIO_compressGzFrame(cRess_t* ress, ret = deflate(&strm, Z_NO_FLUSH); if (ret != Z_OK) EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); - { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; - if (decompBytes) { - if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) - EXM_THROW(73, "Write error : cannot write to output file : %s", strerror(errno)); - outFileSize += decompBytes; + { size_t const cSize = ress->dstBufferSize - strm.avail_out; + if (cSize) { + if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) + EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno)); + outFileSize += cSize; strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; - } - } - if (srcFileSize == UTIL_FILESIZE_UNKNOWN) - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", + } } + if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(inFileSize>>20), (double)outFileSize/inFileSize*100) - else - DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", + } else { + DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ", (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), (double)outFileSize/inFileSize*100); - } + } } while (1) { - ret = deflate(&strm, Z_FINISH); - { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; - if (decompBytes) { - if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) - EXM_THROW(75, "Write error : %s", strerror(errno)); - outFileSize += decompBytes; + int const ret = deflate(&strm, Z_FINISH); + { size_t const cSize = ress->dstBufferSize - strm.avail_out; + if (cSize) { + if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) + EXM_THROW(75, "Write error : %s ", strerror(errno)); + outFileSize += cSize; strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; } } @@ -896,11 +905,11 @@ FIO_compressGzFrame(cRess_t* ress, EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); } - ret = deflateEnd(&strm); - if (ret != Z_OK) - EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); + { int const ret = deflateEnd(&strm); + if (ret != Z_OK) { + EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); + } } *readsize = inFileSize; - return outFileSize; } #endif @@ -923,14 +932,14 @@ FIO_compressLzmaFrame(cRess_t* ress, if (plain_lzma) { lzma_options_lzma opt_lzma; if (lzma_lzma_preset(&opt_lzma, compressionLevel)) - EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName); + EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName); ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ if (ret != LZMA_OK) - EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); + EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); } else { ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ if (ret != LZMA_OK) - EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); + EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); } strm.next_in = 0; @@ -950,11 +959,11 @@ FIO_compressLzmaFrame(cRess_t* ress, ret = lzma_code(&strm, action); if (ret != LZMA_OK && ret != LZMA_STREAM_END) - EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); + EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); { size_t const compBytes = ress->dstBufferSize - strm.avail_out; if (compBytes) { if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) - EXM_THROW(73, "Write error : %s", strerror(errno)); + EXM_THROW(85, "Write error : %s", strerror(errno)); outFileSize += compBytes; strm.next_out = (BYTE*)ress->dstBuffer; strm.avail_out = ress->dstBufferSize; @@ -1493,7 +1502,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con sfnSize = strlen(outDirFilename); assert(outDirFilename != NULL); } - + if (dfnbCapacity <= sfnSize+suffixSize+1) { /* resize buffer for dstName */ free(dstFileNameBuffer); @@ -1522,9 +1531,10 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con * or into one file each (outFileName == NULL, but suffix != NULL), * or into a destination folder (specified with -O) */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, - const char* outDirName, unsigned nbFiles, - const char* outFileName, const char* suffix, +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, + const char** inFileNamesTable, unsigned nbFiles, + const char* outDirName, + const char* outFileName, const char* suffix, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { @@ -1617,7 +1627,11 @@ static void FIO_freeDResources(dRess_t ress) /** FIO_fwriteSparse() : * @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ -static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +static unsigned +FIO_fwriteSparse(const FIO_prefs_t* const prefs, + FILE* file, + const void* buffer, size_t bufferSize, + unsigned storedSkips) { const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ size_t bufferSizeT = bufferSize / sizeof(size_t); @@ -1625,6 +1639,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi const size_t* ptrT = bufferT; static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ + if (prefs->testMode) return 0; /* do not output anything in test mode */ + if (!prefs->sparseFileSupport) { /* normal write */ size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); if (sizeCheck != bufferSize) @@ -1637,7 +1653,7 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi if (storedSkips > 1 GB) { int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR); if (seekResult != 0) - EXM_THROW(71, "1 GB skip error (sparse file support)"); + EXM_THROW(91, "1 GB skip error (sparse file support)"); storedSkips -= 1 GB; } @@ -1653,13 +1669,13 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi if (nb0T != seg0SizeT) { /* not all 0s */ int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); + if (seekResult) EXM_THROW(92, "Sparse skip error ; try --no-sparse"); storedSkips = 0; seg0SizeT -= nb0T; ptrT += nb0T; { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file); if (sizeCheck != seg0SizeT) - EXM_THROW(73, "Write error : cannot write decoded block : %s", + EXM_THROW(93, "Write error : cannot write decoded block : %s", strerror(errno)); } } ptrT += seg0SizeT; @@ -1677,11 +1693,11 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi if (restPtr != restEnd) { int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); if (seekResult) - EXM_THROW(74, "Sparse skip error ; try --no-sparse"); + EXM_THROW(94, "Sparse skip error ; try --no-sparse"); storedSkips = 0; { size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file); if (sizeCheck != (size_t)(restEnd - restPtr)) - EXM_THROW(75, "Write error : cannot write decoded end of block : %s", + EXM_THROW(95, "Write error : cannot write decoded end of block : %s", strerror(errno)); } } } } @@ -1689,8 +1705,9 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi } static void -FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) +FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) { + if (prefs->testMode) assert(storedSkips == 0); if (storedSkips>0) { assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ @@ -1707,7 +1724,7 @@ FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode @return : 0 (no error) */ -static int FIO_passThrough(FIO_prefs_t* const prefs, +static int FIO_passThrough(const FIO_prefs_t* const prefs, FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded) @@ -1747,7 +1764,10 @@ static unsigned FIO_highbit64(unsigned long long v) /* FIO_zstdErrorHelp() : * detailed error message when requested window size is too large */ -static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName) +static void +FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, + const dRess_t* ress, + size_t err, const char* srcFileName) { ZSTD_frameHeader header; @@ -1779,12 +1799,10 @@ static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t er * @return : size of decoded zstd frame, or an error code */ #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) -static unsigned long long FIO_decompressZstdFrame( - FIO_prefs_t* const prefs, - dRess_t* ress, - FILE* finput, - const char* srcFileName, - U64 alreadyDecoded) +static unsigned long long +FIO_decompressZstdFrame(const FIO_prefs_t* const prefs, + dRess_t* ress, FILE* finput, + const char* srcFileName, U64 alreadyDecoded) { U64 frameSize = 0; U32 storedSkips = 0; @@ -1848,13 +1866,16 @@ static unsigned long long FIO_decompressZstdFrame( #ifdef ZSTD_GZDECOMPRESS -static unsigned long long FIO_decompressGzFrame(dRess_t* ress, - FILE* srcFile, const char* srcFileName) +static unsigned long long +FIO_decompressGzFrame(const FIO_prefs_t* const prefs, + dRess_t* ress, FILE* srcFile, + const char* srcFileName) { unsigned long long outFileSize = 0; z_stream strm; int flush = Z_NO_FLUSH; int decodingError = 0; + unsigned storedSkips = 0; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; @@ -1889,10 +1910,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, } { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { - if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { - DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno)); - decodingError = 1; break; - } + storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips); outFileSize += decompBytes; strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; @@ -1909,19 +1927,24 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); decodingError = 1; } + FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } #endif #ifdef ZSTD_LZMADECOMPRESS -static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma) +static unsigned long long +FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs, + dRess_t* ress, FILE* srcFile, + const char* srcFileName, int plain_lzma) { unsigned long long outFileSize = 0; lzma_stream strm = LZMA_STREAM_INIT; lzma_action action = LZMA_RUN; lzma_ret initRet; int decodingError = 0; + unsigned storedSkips = 0; strm.next_in = 0; strm.avail_in = 0; @@ -1964,10 +1987,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, } { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { - if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { - DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno)); - decodingError = 1; break; - } + storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips); outFileSize += decompBytes; strm.next_out = (BYTE*)ress->dstBuffer; strm.avail_out = ress->dstBufferSize; @@ -1979,19 +1999,23 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, memmove(ress->srcBuffer, strm.next_in, strm.avail_in); ress->srcBufferLoaded = strm.avail_in; lzma_end(&strm); + FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; } #endif #ifdef ZSTD_LZ4DECOMPRESS -static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, - FILE* srcFile, const char* srcFileName) +static unsigned long long +FIO_decompressLz4Frame(const FIO_prefs_t* const prefs, + dRess_t* ress, FILE* srcFile, + const char* srcFileName) { unsigned long long filesize = 0; LZ4F_errorCode_t nextToLoad; LZ4F_decompressionContext_t dCtx; LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); int decodingError = 0; + unsigned storedSkips = 0; if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); @@ -2035,10 +2059,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, /* Write Block */ if (decodedBytes) { - if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) { - DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno)); - decodingError = 1; nextToLoad = 0; break; - } + storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decodedBytes, storedSkips); filesize += decodedBytes; DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); } @@ -2059,6 +2080,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, LZ4F_freeDecompressionContext(dCtx); ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ + FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; } @@ -2072,8 +2094,9 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, * @return : 0 : OK * 1 : error */ -static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, - const char* dstFileName, const char* srcFileName) +static int FIO_decompressFrames(const FIO_prefs_t* const prefs, + dRess_t ress, FILE* srcFile, + const char* dstFileName, const char* srcFileName) { unsigned readSomething = 0; unsigned long long filesize = 0; @@ -2105,7 +2128,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr filesize += frameSize; } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ #ifdef ZSTD_GZDECOMPRESS - unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName); + unsigned long long const frameSize = FIO_decompressGzFrame(prefs, &ress, srcFile, srcFileName); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; #else @@ -2115,7 +2138,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ #ifdef ZSTD_LZMADECOMPRESS - unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); + unsigned long long const frameSize = FIO_decompressLzmaFrame(prefs, &ress, srcFile, srcFileName, buf[0] != 0xFD); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; #else @@ -2124,7 +2147,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr #endif } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { #ifdef ZSTD_LZ4DECOMPRESS - unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName); + unsigned long long const frameSize = FIO_decompressLz4Frame(prefs, &ress, srcFile, srcFileName); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; #else @@ -2164,11 +2187,11 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, int transfer_permissions = 0; int releaseDstFile = 0; - if (ress.dstFile == NULL) { + if ((ress.dstFile == NULL) && (prefs->testMode==0)) { releaseDstFile = 1; ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName); - if (ress.dstFile==0) return 1; + if (ress.dstFile==NULL) return 1; /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, @@ -2181,7 +2204,6 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, transfer_permissions = 1; } - result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName); if (releaseDstFile) { @@ -2278,7 +2300,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) char* outDirFilename = NULL; size_t sfnSize = strlen(srcFileName); size_t suffixSize; - + const char* const suffixPtr = strrchr(srcFileName, '.'); if (suffixPtr == NULL) { DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", @@ -2328,7 +2350,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) dfnbCapacity = sfnSize + 20; dstFileNameBuffer = (char*)malloc(dfnbCapacity); if (dstFileNameBuffer==NULL) - EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno)); + EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno)); } /* return dst name == src name truncated from suffix */ @@ -2357,11 +2379,13 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, if (outFileName) { unsigned u; - ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); - if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); + if (!prefs->testMode) { + ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); + if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); + } for (u=0; utestMode) && (fclose(ress.dstFile))) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); } else { diff --git a/programs/fileio.h b/programs/fileio.h index ded1c0373..4b0143beb 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -26,7 +26,7 @@ extern "C" { #define stdinmark "/*stdin*\\" #define stdoutmark "/*stdout*\\" #ifdef _WIN32 -# define nulmark "nul" +# define nulmark "NUL" #else # define nulmark "/dev/null" #endif @@ -74,6 +74,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable); void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize); void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize); void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint); +void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode); void FIO_setLiteralCompressionMode( FIO_prefs_t* const prefs, ZSTD_literalCompressionMode_e mode); @@ -85,14 +86,14 @@ void FIO_setNotificationLevel(int level); * Single File functions ***************************************/ /** FIO_compressFilename() : - @return : 0 == ok; 1 == pb with src file. */ + * @return : 0 == ok; 1 == pb with src file. */ int FIO_compressFilename (FIO_prefs_t* const prefs, const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams); /** FIO_decompressFilename() : - @return : 0 == ok; 1 == pb with src file. */ + * @return : 0 == ok; 1 == pb with src file. */ int FIO_decompressFilename (FIO_prefs_t* const prefs, const char* outfilename, const char* infilename, const char* dictFileName); @@ -103,15 +104,16 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis * Multiple File functions ***************************************/ /** FIO_compressMultipleFilenames() : - @return : nb of missing files */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, - const char* outDirName, unsigned nbFiles, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, + * @return : nb of missing files */ +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, + const char** inFileNamesTable, unsigned nbFiles, + const char* outDirName, + const char* outFileName, const char* suffix, + const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams); /** FIO_decompressMultipleFilenames() : - @return : nb of missing or skipped files */ + * @return : nb of missing or skipped files */ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char** srcNamesTable, unsigned nbFiles, const char* outDirName, @@ -119,10 +121,12 @@ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char* dictFileName); /* FIO_checkFilenameCollisions() : - * Checks for and warns if thereƄ are any files that would have the same output path + * Checks for and warns if there are any files that would have the same output path */ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles); + + /*-************************************* * Advanced stuff (should actually be hosted elsewhere) ***************************************/ diff --git a/programs/util.c b/programs/util.c index 0908a43a2..e1aa447a2 100644 --- a/programs/util.c +++ b/programs/util.c @@ -106,20 +106,23 @@ int UTIL_compareStr(const void *p1, const void *p2) { return strcmp(* (char * const *) p1, * (char * const *) p2); } -int UTIL_isSameFile(const char* file1, const char* file2) +int UTIL_isSameFile(const char* fName1, const char* fName2) { -#if defined(_MSC_VER) + assert(fName1 != NULL); assert(fName2 != NULL); +#if defined(_MSC_VER) || defined(_WIN32) /* note : Visual does not support file identification by inode. + * inode does not work on Windows, even with a posix layer, like msys2. * The following work-around is limited to detecting exact name repetition only, * aka `filename` is considered different from `subdir/../filename` */ - return !strcmp(file1, file2); + return !strcmp(fName1, fName2); #else - stat_t file1Stat; - stat_t file2Stat; - return UTIL_getFileStat(file1, &file1Stat) - && UTIL_getFileStat(file2, &file2Stat) - && (file1Stat.st_dev == file2Stat.st_dev) - && (file1Stat.st_ino == file2Stat.st_ino); + { stat_t file1Stat; + stat_t file2Stat; + return UTIL_getFileStat(fName1, &file1Stat) + && UTIL_getFileStat(fName2, &file2Stat) + && (file1Stat.st_dev == file2Stat.st_dev) + && (file1Stat.st_ino == file2Stat.st_ino); + } #endif } @@ -240,19 +243,20 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char DIR *dir; struct dirent *entry; char* path; - int dirLength, fnameLength, pathLength, nbFiles = 0; + size_t dirLength, fnameLength, pathLength; + int nbFiles = 0; if (!(dir = opendir(dirName))) { UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); return 0; } - dirLength = (int)strlen(dirName); + dirLength = strlen(dirName); errno = 0; while ((entry = readdir(dir)) != NULL) { if (strcmp (entry->d_name, "..") == 0 || strcmp (entry->d_name, ".") == 0) continue; - fnameLength = (int)strlen(entry->d_name); + fnameLength = strlen(entry->d_name); path = (char*) malloc(dirLength + fnameLength + 2); if (!path) { closedir(dir); return 0; } memcpy(path, dirName, dirLength); @@ -274,7 +278,8 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char } else { if (*bufStart + *pos + pathLength >= *bufEnd) { ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; - *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); + assert(newListSize >= 0); + *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize); *bufEnd = *bufStart + newListSize; if (*bufStart == NULL) { free(path); closedir(dir); return 0; } } @@ -323,7 +328,6 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, unsigned i, nbFiles; char* buf = (char*)malloc(LIST_SIZE_INCREASE); char* bufend = buf + LIST_SIZE_INCREASE; - const char** fileTable; if (!buf) return NULL; @@ -332,36 +336,37 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, size_t const len = strlen(inputNames[i]); if (buf + pos + len >= bufend) { ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; - buf = (char*)UTIL_realloc(buf, newListSize); + assert(newListSize >= 0); + buf = (char*)UTIL_realloc(buf, (size_t)newListSize); bufend = buf + newListSize; if (!buf) return NULL; } if (buf + pos + len < bufend) { - memcpy(buf+pos, inputNames[i], len+1); /* with final \0 */ + memcpy(buf+pos, inputNames[i], len+1); /* including final \0 */ pos += len + 1; nbFiles++; } } else { - nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks); + nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks); if (buf == NULL) return NULL; } } if (nbFiles == 0) { free(buf); return NULL; } - fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*)); - if (!fileTable) { free(buf); return NULL; } + { const char** const fileTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileTable)); + if (!fileTable) { free(buf); return NULL; } - for (i=0, pos=0; i bufend) { free(buf); free((void*)fileTable); return NULL; } + pos += strlen(fileTable[i]) + 1; + } + + *allocatedBuffer = buf; + *allocatedNamesNb = nbFiles; + + return fileTable; } - - if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; } - - *allocatedBuffer = buf; - *allocatedNamesNb = nbFiles; - - return fileTable; } @@ -394,8 +399,13 @@ int UTIL_countPhysicalCores(void) DWORD returnLength = 0; size_t byteOffset = 0; - glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")), - "GetLogicalProcessorInformation"); +#if defined(_MSC_VER) +/* Visual Studio does not like the following cast */ +# pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */ +# pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */ +#endif + glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")), + "GetLogicalProcessorInformation"); if (glpi == NULL) { goto failed; diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res index 843499254..9984215b5 100644 Binary files a/programs/windres/zstd32.res and b/programs/windres/zstd32.res differ diff --git a/programs/windres/zstd64.res b/programs/windres/zstd64.res index da859810f..615f38944 100644 Binary files a/programs/windres/zstd64.res and b/programs/windres/zstd64.res differ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index ae53d2c39..aed9ee867 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -287,7 +287,7 @@ static unsigned readU32FromChar(const char** stringPtr) { * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. * @return 0 and doesn't modify *stringPtr otherwise. */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +static int longCommandWArg(const char** stringPtr, const char* longCommand) { size_t const comSize = strlen(longCommand); int const result = !strncmp(*stringPtr, longCommand, comSize); @@ -430,8 +430,8 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void) static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr) { for ( ; ;) { - if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } - if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } DISPLAYLEVEL(4, "invalid compression parameter \n"); return 0; } @@ -526,7 +526,7 @@ static int init_cLevel(void) { DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env); return ZSTDCLI_CLEVEL_DEFAULT; } else if (*ptr == 0) { - return sign * absLevel; + return sign * (int)absLevel; } } @@ -584,7 +584,7 @@ int main(int argCount, const char* argv[]) int cLevelLast = -1000000000; unsigned recursive = 0; unsigned memLimit = 0; - const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ + const char** filenameTable = (const char**)malloc((size_t)argCount * sizeof(const char*)); /* argCount >= 1 */ unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; @@ -745,7 +745,7 @@ int main(int argCount, const char* argv[]) continue; } #endif - if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } @@ -807,7 +807,7 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOCOMPRESS /* compression Level */ if ((*argument>='0') && (*argument<='9')) { - dictCLevel = cLevel = readU32FromChar(&argument); + dictCLevel = cLevel = (int)readU32FromChar(&argument); continue; } #endif @@ -856,7 +856,7 @@ int main(int argCount, const char* argv[]) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; - + /* limit decompression memory */ case 'M': argument++; @@ -879,7 +879,7 @@ int main(int argCount, const char* argv[]) case 'e': /* compression Level */ argument++; - cLevelLast = readU32FromChar(&argument); + cLevelLast = (int)readU32FromChar(&argument); break; /* Modify Nb Iterations (benchmark only) */ @@ -905,7 +905,7 @@ int main(int argCount, const char* argv[]) /* nb of threads (hidden option) */ case 'T': argument++; - nbWorkers = readU32FromChar(&argument); + nbWorkers = (int)readU32FromChar(&argument); break; /* Dictionary Selection level */ @@ -1042,16 +1042,16 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOBENCH benchParams.blockSize = blockSize; benchParams.nbWorkers = nbWorkers; - benchParams.realTime = setRealTimePrio; + benchParams.realTime = (unsigned)setRealTimePrio; benchParams.nbSeconds = bench_nbSeconds; benchParams.ldmFlag = ldmFlag; - benchParams.ldmMinMatch = g_ldmMinMatch; - benchParams.ldmHashLog = g_ldmHashLog; + benchParams.ldmMinMatch = (int)g_ldmMinMatch; + benchParams.ldmHashLog = (int)g_ldmHashLog; if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { - benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog; + benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog; } if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) { - benchParams.ldmHashRateLog = g_ldmHashRateLog; + benchParams.ldmHashRateLog = (int)g_ldmHashRateLog; } benchParams.literalCompressionMode = literalCompressionMode; @@ -1092,16 +1092,16 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NODICT ZDICT_params_t zParams; zParams.compressionLevel = dictCLevel; - zParams.notificationLevel = g_displayLevel; + zParams.notificationLevel = (unsigned)g_displayLevel; zParams.dictID = dictID; if (dict == cover) { int const optimize = !coverParams.k || !coverParams.d; - coverParams.nbThreads = nbWorkers; + coverParams.nbThreads = (unsigned)nbWorkers; coverParams.zParams = zParams; operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize); } else if (dict == fastCover) { int const optimize = !fastCoverParams.k || !fastCoverParams.d; - fastCoverParams.nbThreads = nbWorkers; + fastCoverParams.nbThreads = (unsigned)nbWorkers; fastCoverParams.zParams = zParams; operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize); } else { @@ -1120,7 +1120,7 @@ int main(int argCount, const char* argv[]) } #ifndef ZSTD_NODECOMPRESS - if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */ + if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */ #endif /* No input filename ==> use stdin and stdout */ @@ -1156,14 +1156,14 @@ int main(int argCount, const char* argv[]) if (operation==zom_compress) { #ifndef ZSTD_NOCOMPRESS FIO_setNbWorkers(prefs, nbWorkers); - FIO_setBlockSize(prefs, (U32)blockSize); - if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog); - FIO_setLdmFlag(prefs, ldmFlag); - FIO_setLdmHashLog(prefs, g_ldmHashLog); - FIO_setLdmMinMatch(prefs, g_ldmMinMatch); - if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog); - if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog); - FIO_setAdaptiveMode(prefs, adapt); + FIO_setBlockSize(prefs, (int)blockSize); + if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog); + FIO_setLdmFlag(prefs, (unsigned)ldmFlag); + FIO_setLdmHashLog(prefs, (int)g_ldmHashLog); + FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch); + if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog); + if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog); + FIO_setAdaptiveMode(prefs, (unsigned)adapt); FIO_setAdaptMin(prefs, adaptMin); FIO_setAdaptMax(prefs, adaptMax); FIO_setRsyncable(prefs, rsyncable); @@ -1177,7 +1177,7 @@ int main(int argCount, const char* argv[]) if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); diff --git a/tests/Makefile b/tests/Makefile index bd2f90976..161c823ed 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -250,7 +250,8 @@ clean: $(MAKE) -C $(ZSTDDIR) clean $(MAKE) -C $(PRGDIR) clean @$(RM) -fR $(TESTARTEFACT) - @$(RM) -f core *.o tmp* *.tmp result* *.gcda dictionary *.zst \ + @$(RM) -rf tmp* # some test directories are named tmp* + @$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \ $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fullbench-lib$(EXT) fullbench-dll$(EXT) \ @@ -264,7 +265,7 @@ clean: #---------------------------------------------------------------------------------- -#make valgrindTest is validated only for Linux, macOS, BSD, Hurd and Solaris targets +# valgrind tests are validated only for some posix platforms #---------------------------------------------------------------------------------- ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) HOST_OS = POSIX @@ -286,7 +287,7 @@ valgrindTest: zstd datagen fuzzer fullbench endif -ifneq (,$(filter MSYS%,$(shell uname))) +ifneq (,$(filter MINGW% MSYS%,$(shell uname))) HOST_OS = MSYS endif diff --git a/tests/playTests.sh b/tests/playTests.sh index 06de4f8ac..c74d548b8 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -64,11 +64,12 @@ PRGDIR="$SCRIPT_DIR/../programs" TESTDIR="$SCRIPT_DIR/../tests" UNAME=$(uname) -isTerminal=false +detectedTerminal=false if [ -t 0 ] && [ -t 1 ] then - isTerminal=true + detectedTerminal=true fi +isTerminal=${isTerminal:-$detectedTerminal} isWindows=false INTOVOID="/dev/null" @@ -211,8 +212,8 @@ $ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t $ZSTD tmp -c --compress-literals -1 | $ZSTD -t $ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t $ZSTD tmp -c --compress-literals -19 | $ZSTD -t -$ZSTD -b --fast=1 -i1e1 tmp --compress-literals -$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals +$ZSTD -b --fast=1 -i0e1 tmp --compress-literals +$ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals println "test : file removal" $ZSTD -f --rm tmp @@ -239,7 +240,7 @@ rm tmp # erase source file touch tmp.zst # create destination file $ZSTD -f tmp && die "attempt to compress a non existing file" test -f tmp.zst # destination file should still be present -rm tmp* +rm -rf tmp* # may also erase tmp* directory from previous failed run println "\n===> decompression only tests " head -c 1048576 /dev/zero > tmp @@ -283,7 +284,7 @@ test -f tmpOutDir/tmp1.zst test -f tmpOutDir/tmp2.zst println "test : decompress multiple files into an output directory, --output-dir-flat" mkdir tmpOutDirDecomp -$ZSTD tmpOutDir/ -r -d --output-dir-flat tmpOutDirDecomp +$ZSTD tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp test -f tmpOutDirDecomp/tmp2 test -f tmpOutDirDecomp/tmp1 rm -rf tmp* @@ -577,30 +578,27 @@ println "- Create dictionary with short dictID" $ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" println "- Create dictionaries with shrink-dict flag enabled" -$ZSTD --train-fastcover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict -$ZSTD --train-fastcover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 -$ZSTD --train-fastcover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 +$ZSTD --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict +$ZSTD --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 +$ZSTD --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 println "- Create dictionary with size limit" -$ZSTD --train-fastcover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -println "- Compare size of dictionary from 90% training samples with 80% training samples" -$ZSTD --train-fastcover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c -$ZSTD --train-fastcover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +$ZSTD --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K println "- Create dictionary using all samples for both training and testing" -$ZSTD --train-fastcover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +$ZSTD --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c println "- Create dictionary using f=16" -$ZSTD --train-fastcover=f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c -$ZSTD --train-fastcover=accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" +$ZSTD --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +$ZSTD --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" println "- Create dictionary using accel=2" -$ZSTD --train-fastcover=accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +$ZSTD --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c println "- Create dictionary using accel=10" -$ZSTD --train-fastcover=accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +$ZSTD --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c println "- Create dictionary with multithreading" $ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c println "- Test -o before --train-fastcover" rm -f tmpDict dictionary -$ZSTD -o tmpDict --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c +$ZSTD -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c test -f tmpDict -$ZSTD --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c +$ZSTD --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c test -f dictionary rm tmp* dictionary @@ -674,10 +672,10 @@ $ZSTD -i0b0e3 tmp1 println "bench negative level" $ZSTD -bi0 --fast tmp1 println "with recursive and quiet modes" -$ZSTD -rqi1b1e2 tmp1 +$ZSTD -rqi0b1e2 tmp1 println "benchmark decompression only" $ZSTD -f tmp1 -$ZSTD -b -d -i1 tmp1.zst +$ZSTD -b -d -i0 tmp1.zst println "\n===> zstd compatibility tests "