diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index 8aa37650e..d3275b7df 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -1,6 +1,6 @@ /* Buffered version of Zstd compression library - Copyright (C) 2015, Yann Collet. + Copyright (C) 2015-2016, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -26,14 +26,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c + - zstd homepage : http://www.zstd.net/ */ #ifndef ZSTD_BUFFERED_H #define ZSTD_BUFFERED_H /* The objects defined into this file should be considered experimental. - * They are not labelled stable, as their prototype may change in the future. + * They are not considered stable, as their prototype may change in the future. * You can use them for tests, provide feedback, or if you can endure risk of future changes. */ @@ -42,7 +41,7 @@ extern "C" { #endif /* ************************************* -* Includes +* Dependencies ***************************************/ #include /* size_t */ @@ -75,7 +74,7 @@ ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* d ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); -/** ************************************************ +/*-************************************************* * Streaming compression * * A ZBUFF_CCtx object is required to track streaming operation. @@ -123,12 +122,14 @@ ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); +ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr); -/** ************************************************ +/*-*************************************************************************** * Streaming decompression * -* A ZBUFF_DCtx object is required to track streaming operation. +* A ZBUFF_DCtx object is required to track streaming operations. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. * Use ZBUFF_decompressInit() to start a new decompression operation, * or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. @@ -143,10 +144,10 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* * or 0 when a frame is completely decoded * or an error code, which can be tested using ZBUFF_isError(). * -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. -* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* **************************************************/ +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() / ZBUFF_recommendedDOutSize() +* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ /* ************************************* @@ -155,7 +156,7 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); -/** The below functions provide recommended buffer sizes for Compression or Decompression operations. +/** Functions below provide recommended buffer sizes for Compression or Decompression operations. * These sizes are just hints, and tend to offer better latency */ ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); diff --git a/lib/zstd_buffered_static.h b/lib/zstd_buffered_static.h index 5052f4c3e..a15745c09 100644 --- a/lib/zstd_buffered_static.h +++ b/lib/zstd_buffered_static.h @@ -1,7 +1,7 @@ /* zstd - buffered version of compression library experimental complementary API, for static linking only - Copyright (C) 2015, Yann Collet. + Copyright (C) 2015-2016, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -27,8 +27,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c + - zstd homepage : http://www.zstd.net */ #ifndef ZSTD_BUFFERED_STATIC_H #define ZSTD_BUFFERED_STATIC_H diff --git a/programs/fileio.c b/programs/fileio.c index a5739ab4d..3e467fe29 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -186,54 +186,73 @@ static U64 FIO_getFileSize(const char* infilename) } +static FILE* FIO_openSrcFile(const char* srcFileName) +{ + FILE* f; + + if (!strcmp (srcFileName, stdinmark)) { + DISPLAYLEVEL(4,"Using stdin for input\n"); + f = stdin; + SET_BINARY_MODE(stdin); + } else { + f = fopen(srcFileName, "rb"); + } + + if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: No such file\n", srcFileName); + + return f; +} + + +static FILE* FIO_openDstFile(const char* dstFileName) +{ + FILE* f; + + if (!strcmp (dstFileName, stdoutmark)) { + DISPLAYLEVEL(4,"Using stdout for output\n"); + f = stdout; + SET_BINARY_MODE(stdout); + } else { + if (!g_overwrite) { /* Check if destination file already exists */ + f = fopen( dstFileName, "rb" ); + if (f != 0) { /* dest file exists, prompt for overwrite authorization */ + fclose(f); + if (g_displayLevel <= 1) { + /* No interaction possible */ + DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName); + return 0; + } + DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName); + { + int ch = getchar(); + if ((ch!='Y') && (ch!='y')) { + DISPLAY(" not overwritten \n"); + return 0; + } + while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ + } } } + f = fopen( dstFileName, "wb" ); + } + return f; +} + + static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr, const char* dstFileName, const char* srcFileName) { - if (!strcmp (srcFileName, stdinmark)) { - DISPLAYLEVEL(4,"Using stdin for input\n"); - *fileInPtr = stdin; - SET_BINARY_MODE(stdin); - } else { - *fileInPtr = fopen(srcFileName, "rb"); - } - + *fileInPtr = FIO_openSrcFile(srcFileName); if ( *fileInPtr==0 ) { DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName); return 1; } - if (!strcmp (dstFileName, stdoutmark)) { - DISPLAYLEVEL(4,"Using stdout for output\n"); - *fileOutPtr = stdout; - SET_BINARY_MODE(stdout); - } else { - if (!g_overwrite) { /* Check if destination file already exists */ - *fileOutPtr = fopen( dstFileName, "rb" ); - if (*fileOutPtr != 0) { /* dest file exists, prompt for overwrite authorization */ - fclose(*fileOutPtr); - DISPLAY("Warning : %s already exists \n", dstFileName); - if ((g_displayLevel <= 1) || (*fileInPtr == stdin)) { - /* No interaction possible */ - DISPLAY("Operation aborted : %s already exists \n", dstFileName); - return 1; - } - DISPLAY("Overwrite ? (y/N) : "); - { - int ch = getchar(); - if ((ch!='Y') && (ch!='y')) { - DISPLAY("No. Operation aborted : %s already exists \n", dstFileName); - return 1; - } - while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ - } } } - *fileOutPtr = fopen( dstFileName, "wb" ); - } - - if (*fileOutPtr==0) EXM_THROW(13, "Pb opening %s", dstFileName); + *fileOutPtr = FIO_openDstFile(dstFileName); + if (*fileOutPtr==0) return 1; return 0; } + /*!FIO_loadFile * creates a buffer, pointed by *bufferPtr, * loads "filename" content into it @@ -280,6 +299,7 @@ typedef struct { void* dictBuffer; size_t dictBufferSize; ZBUFF_CCtx* ctx; + FILE* dstFile; } cRess_t; static cRess_t FIO_createCResources(const char* dictFileName) @@ -313,24 +333,25 @@ static void FIO_freeCResources(cRess_t ress) } -/*! - * FIO_compressFilename_extRess() : - * @result : 0 : compression completed correctly, - * 1 : missing or pb opening srcFileName +/*! FIO_compressFilename_internal() : + * same as FIO_compressFilename_extRess(), with ress.desFile already opened + * @return : 0 : compression completed correctly, + * 1 : missing or pb opening srcFileName */ -static int FIO_compressFilename_extRess(cRess_t ress, - const char* dstFileName, const char* srcFileName, - int cLevel) +static int FIO_compressFilename_internal(cRess_t ress, + const char* dstFileName, const char* srcFileName, + int cLevel) { FILE* srcFile; - FILE* dstFile; + FILE* dstFile = ress.dstFile; U64 filesize = 0; U64 compressedfilesize = 0; size_t dictSize = ress.dictBufferSize; size_t sizeCheck, errorCode; /* File check */ - if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1; + srcFile = FIO_openSrcFile(srcFileName); + if (!srcFile) return 1; /* srcFile could not be opened */ /* init */ filesize = FIO_getFileSize(srcFileName) + dictSize; @@ -346,8 +367,7 @@ static int FIO_compressFilename_extRess(cRess_t ress, filesize += inSize; DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20)); - { - /* Compress (buffered streaming ensures appropriate formatting) */ + { /* Compress using buffered streaming */ size_t usedInSize = inSize; size_t cSize = ress.dstBufferSize; size_t result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize); @@ -362,7 +382,6 @@ static int FIO_compressFilename_extRess(cRess_t ress, if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); compressedfilesize += cSize; } - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100); } @@ -384,12 +403,31 @@ static int FIO_compressFilename_extRess(cRess_t ress, /* clean */ fclose(srcFile); - if (fclose(dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName); return 0; } +/*! FIO_compressFilename_extRess() : + * @return : 0 : compression completed correctly, + * 1 : missing or pb opening srcFileName + */ +static int FIO_compressFilename_extRess(cRess_t ress, + const char* dstFileName, const char* srcFileName, + int cLevel) +{ + int result; + + ress.dstFile = FIO_openDstFile(dstFileName); + if (ress.dstFile==0) return 1; + + result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); + + if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName); + return result; +} + + int FIO_compressFilename(const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel) { @@ -427,20 +465,28 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile int missed_files = 0; char* dstFileName = (char*)malloc(FNSPACE); size_t dfnSize = FNSPACE; - const size_t suffixSize = strlen(suffix); + const size_t suffixSize = suffix ? strlen(suffix) : 0; cRess_t ress; /* init */ ress = FIO_createCResources(dictFileName); /* loop on each file */ - for (u=0; u hello.tmp echo "world!" > world.tmp @@ -37,7 +38,8 @@ rm ./*.tmp ./*.zstd echo frame concatenation test completed -echo "**** flush write error test **** " + +echo "\n**** flush write error test **** " echo "echo foo | $ZSTD > /dev/full" echo foo | $ZSTD > /dev/full && die "write error not detected!" @@ -45,27 +47,32 @@ echo "echo foo | $ZSTD | $ZSTD -d > /dev/full" echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!" -echo "*** dictionary tests *** " +echo "\n**** dictionary tests **** " ./datagen > tmpDict ./datagen -g1M | md5sum > tmp1 -./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dv | md5sum > tmp2 +./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2 diff -q tmp1 tmp2 -echo "*** multiple files tests *** " +echo "\n**** multiple files tests **** " ./datagen -s1 > tmp1 2> /dev/null ./datagen -s2 -g100K > tmp2 2> /dev/null ./datagen -s3 -g1M > tmp3 2> /dev/null $ZSTD -f tmp* +echo "compress tmp* : " ls -ls tmp* rm tmp1 tmp2 tmp3 +echo "decompress tmp* : " $ZSTD -df *.zst ls -ls tmp* +echo "compress tmp* into stdout : " +$ZSTD -c tmp1 tmp2 tmp3 > tmpall +ls -ls tmp* $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!" rm tmp* -echo "**** zstd round-trip tests **** " +echo "\n**** zstd round-trip tests **** " roundTripTest roundTripTest '' 6 diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 41083eb63..f481b7d8f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -379,28 +379,28 @@ int main(int argCount, const char** argv) if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName); if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) return badusage(programName); - /* No warning message in pipe mode (stdin + stdout) or multiple mode */ - if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1; - if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1; - - /* user-selected output filename only possible with a single file */ - if ((outFileName) && (filenameIdx>1)) { + /* user-selected output filename, only possible with a single file */ + if (outFileName && strcmp(outFileName,stdoutmark) && (filenameIdx>1)) { DISPLAY("Too many files (%u) on the command line. \n", filenameIdx); return filenameIdx; } + /* No warning message in pipe mode (stdin + stdout) or multiple mode */ + if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1; + if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1; + /* IO Stream/File */ FIO_setNotificationLevel(displayLevel); if (decode) { - if (outFileName) + if (filenameIdx==1) operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); else operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName); } else { /* compression */ - if (outFileName) + if (filenameIdx==1) operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel); else - operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName, cLevel); + operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, forceStdout ? NULL : ZSTD_EXTENSION, dictFileName, cLevel); } _end: