lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-01 09:47:01 +03:00
Code Activity

cli : added ability to compress multiple files into stdout (-c)

Browse Source
This commit is contained in:
Yann Collet
2016-02-12 15:56:46 +01:00
parent 492a9bb88e
commit f062436fa6
6 changed files with 144 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
lib/zstd_buffered.h
View File

@ -1,6 +1,6 @@
/*
Buffered version of Zstd compression library
Copyright (C) 2015, Yann Collet.
Copyright (C) 2015-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@ -26,14 +26,13 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
- zstd homepage : http://www.zstd.net/
*/
#ifndef ZSTD_BUFFERED_H
#define ZSTD_BUFFERED_H
/* The objects defined into this file should be considered experimental.
* They are not labelled stable, as their prototype may change in the future.
* They are not considered stable, as their prototype may change in the future.
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
*/
@ -42,7 +41,7 @@ extern "C" {
#endif
/* *************************************
* Includes
* Dependencies
***************************************/
#include <stddef.h> /* size_t */
@ -75,7 +74,7 @@ ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* d
ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
/** ************************************************
/*-*************************************************
* Streaming compression
*
* A ZBUFF_CCtx object is required to track streaming operation.
@ -123,12 +122,14 @@ ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr);
/** ************************************************
/*-***************************************************************************
* Streaming decompression
*
* A ZBUFF_DCtx object is required to track streaming operation.
* A ZBUFF_DCtx object is required to track streaming operations.
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
* Use ZBUFF_decompressInit() to start a new decompression operation,
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
@ -143,10 +144,10 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t*
* or 0 when a frame is completely decoded
* or an error code, which can be tested using ZBUFF_isError().
*
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* **************************************************/
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() / ZBUFF_recommendedDOutSize()
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
/* *************************************
@ -155,7 +156,7 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t*
ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
/** The below functions provide recommended buffer sizes for Compression or Decompression operations.
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, and tend to offer better latency */
ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);

5
lib/zstd_buffered_static.h
View File

@ -1,7 +1,7 @@
/*
zstd - buffered version of compression library
experimental complementary API, for static linking only
Copyright (C) 2015, Yann Collet.
Copyright (C) 2015-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@ -27,8 +27,7 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
- zstd homepage : http://www.zstd.net
*/
#ifndef ZSTD_BUFFERED_STATIC_H
#define ZSTD_BUFFERED_STATIC_H

160
programs/fileio.c
View File

@ -186,54 +186,73 @@ static U64 FIO_getFileSize(const char* infilename)
}
static FILE* FIO_openSrcFile(const char* srcFileName)
{
FILE* f;
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n");
f = stdin;
SET_BINARY_MODE(stdin);
} else {
f = fopen(srcFileName, "rb");
}
if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: No such file\n", srcFileName);
return f;
}
static FILE* FIO_openDstFile(const char* dstFileName)
{
FILE* f;
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n");
f = stdout;
SET_BINARY_MODE(stdout);
} else {
if (!g_overwrite) { /* Check if destination file already exists */
f = fopen( dstFileName, "rb" );
if (f != 0) { /* dest file exists, prompt for overwrite authorization */
fclose(f);
if (g_displayLevel <= 1) {
/* No interaction possible */
DISPLAY("zstd: %s already exists; not overwritten \n", dstFileName);
return 0;
}
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
{
int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n");
return 0;
}
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
} } }
f = fopen( dstFileName, "wb" );
}
return f;
}
static int FIO_getFiles(FILE** fileOutPtr, FILE** fileInPtr,
const char* dstFileName, const char* srcFileName)
{
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input\n");
*fileInPtr = stdin;
SET_BINARY_MODE(stdin);
} else {
*fileInPtr = fopen(srcFileName, "rb");
}
*fileInPtr = FIO_openSrcFile(srcFileName);
if ( *fileInPtr==0 ) {
DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", srcFileName);
return 1;
}
if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output\n");
*fileOutPtr = stdout;
SET_BINARY_MODE(stdout);
} else {
if (!g_overwrite) { /* Check if destination file already exists */
*fileOutPtr = fopen( dstFileName, "rb" );
if (*fileOutPtr != 0) { /* dest file exists, prompt for overwrite authorization */
fclose(*fileOutPtr);
DISPLAY("Warning : %s already exists \n", dstFileName);
if ((g_displayLevel <= 1) || (*fileInPtr == stdin)) {
/* No interaction possible */
DISPLAY("Operation aborted : %s already exists \n", dstFileName);
return 1;
}
DISPLAY("Overwrite ? (y/N) : ");
{
int ch = getchar();
if ((ch!='Y') && (ch!='y')) {
DISPLAY("No. Operation aborted : %s already exists \n", dstFileName);
return 1;
}
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
} } }
*fileOutPtr = fopen( dstFileName, "wb" );
}
if (*fileOutPtr==0) EXM_THROW(13, "Pb opening %s", dstFileName);
*fileOutPtr = FIO_openDstFile(dstFileName);
if (*fileOutPtr==0) return 1;
return 0;
}
/*!FIO_loadFile
* creates a buffer, pointed by *bufferPtr,
* loads "filename" content into it
@ -280,6 +299,7 @@ typedef struct {
void* dictBuffer;
size_t dictBufferSize;
ZBUFF_CCtx* ctx;
FILE* dstFile;
} cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName)
@ -313,24 +333,25 @@ static void FIO_freeCResources(cRess_t ress)
}
/*!
* FIO_compressFilename_extRess() :
* @result : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
/*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess(), with ress.desFile already opened
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
static int FIO_compressFilename_extRess(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
static int FIO_compressFilename_internal(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
{
FILE* srcFile;
FILE* dstFile;
FILE* dstFile = ress.dstFile;
U64 filesize = 0;
U64 compressedfilesize = 0;
size_t dictSize = ress.dictBufferSize;
size_t sizeCheck, errorCode;
/* File check */
if (FIO_getFiles(&dstFile, &srcFile, dstFileName, srcFileName)) return 1;
srcFile = FIO_openSrcFile(srcFileName);
if (!srcFile) return 1; /* srcFile could not be opened */
/* init */
filesize = FIO_getFileSize(srcFileName) + dictSize;
@ -346,8 +367,7 @@ static int FIO_compressFilename_extRess(cRess_t ress,
filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
{
/* Compress (buffered streaming ensures appropriate formatting) */
{ /* Compress using buffered streaming */
size_t usedInSize = inSize;
size_t cSize = ress.dstBufferSize;
size_t result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
@ -362,7 +382,6 @@ static int FIO_compressFilename_extRess(cRess_t ress,
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
compressedfilesize += cSize;
}
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
}
@ -384,12 +403,31 @@ static int FIO_compressFilename_extRess(cRess_t ress,
/* clean */
fclose(srcFile);
if (fclose(dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
return 0;
}
/*! FIO_compressFilename_extRess() :
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
static int FIO_compressFilename_extRess(cRess_t ress,
const char* dstFileName, const char* srcFileName,
int cLevel)
{
int result;
ress.dstFile = FIO_openDstFile(dstFileName);
if (ress.dstFile==0) return 1;
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
return result;
}
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel)
{
@ -427,20 +465,28 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
int missed_files = 0;
char* dstFileName = (char*)malloc(FNSPACE);
size_t dfnSize = FNSPACE;
const size_t suffixSize = strlen(suffix);
const size_t suffixSize = suffix ? strlen(suffix) : 0;
cRess_t ress;
/* init */
ress = FIO_createCResources(dictFileName);
/* loop on each file */
for (u=0; u<nbFiles; u++) {
size_t ifnSize = strlen(inFileNamesTable[u]);
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
strcpy(dstFileName, inFileNamesTable[u]);
strcat(dstFileName, suffix);
missed_files += FIO_compressFilename_extRess(ress, dstFileName, inFileNamesTable[u], compressionLevel);
if (suffix) {
for (u=0; u<nbFiles; u++) {
size_t ifnSize = strlen(inFileNamesTable[u]);
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
strcpy(dstFileName, inFileNamesTable[u]);
strcat(dstFileName, suffix);
missed_files += FIO_compressFilename_extRess(ress, dstFileName,
inFileNamesTable[u], compressionLevel);
}
} else {
ress.dstFile = stdout;
for (u=0; u<nbFiles; u++)
missed_files += FIO_compressFilename_internal(ress, stdoutmark,
inFileNamesTable[u], compressionLevel);
if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
}
/* Close & Free */

3
programs/fileio.h
View File

@ -64,13 +64,14 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con
* Multiple File functions
***************************************/
/** FIO_compressMultipleFilenames() :
if `suffix == NULL`, output is stdout.
@return : nb of missing files */
int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
const char* suffix,
const char* dictFileName, int compressionLevel);
/** FIO_decompressMultipleFilenames() :
@result : nb of missing or skipped files */
@return : nb of missing or skipped files */
int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles,
const char* suffix,
const char* dictFileName);

19
programs/playTests.sh
View File

@ -22,7 +22,8 @@ roundTripTest() {
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
printf "\n**** frame concatenation **** "
echo "\n**** frame concatenation **** "
echo "hello " > hello.tmp
echo "world!" > world.tmp
@ -37,7 +38,8 @@ rm ./*.tmp ./*.zstd
echo frame concatenation test completed
echo "**** flush write error test **** "
echo "\n**** flush write error test **** "
echo "echo foo | $ZSTD > /dev/full"
echo foo | $ZSTD > /dev/full && die "write error not detected!"
@ -45,27 +47,32 @@ echo "echo foo | $ZSTD | $ZSTD -d > /dev/full"
echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
echo "*** dictionary tests *** "
echo "\n**** dictionary tests **** "
./datagen > tmpDict
./datagen -g1M | md5sum > tmp1
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dv | md5sum > tmp2
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
diff -q tmp1 tmp2
echo "*** multiple files tests *** "
echo "\n**** multiple files tests **** "
./datagen -s1 > tmp1 2> /dev/null
./datagen -s2 -g100K > tmp2 2> /dev/null
./datagen -s3 -g1M > tmp3 2> /dev/null
$ZSTD -f tmp*
echo "compress tmp* : "
ls -ls tmp*
rm tmp1 tmp2 tmp3
echo "decompress tmp* : "
$ZSTD -df *.zst
ls -ls tmp*
echo "compress tmp* into stdout : "
$ZSTD -c tmp1 tmp2 tmp3 > tmpall
ls -ls tmp*
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
rm tmp*
echo "**** zstd round-trip tests **** "
echo "\n**** zstd round-trip tests **** "
roundTripTest
roundTripTest '' 6

18
programs/zstdcli.c
View File

@ -379,28 +379,28 @@ int main(int argCount, const char** argv)
if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) return badusage(programName);
/* No warning message in pipe mode (stdin + stdout) or multiple mode */
if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1;
/* user-selected output filename only possible with a single file */
if ((outFileName) && (filenameIdx>1)) {
/* user-selected output filename, only possible with a single file */
if (outFileName && strcmp(outFileName,stdoutmark) && (filenameIdx>1)) {
DISPLAY("Too many files (%u) on the command line. \n", filenameIdx);
return filenameIdx;
}
/* No warning message in pipe mode (stdin + stdout) or multiple mode */
if (!strcmp(filenameTable[0], stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
if ((filenameIdx>1) && (displayLevel==2)) displayLevel=1;
/* IO Stream/File */
FIO_setNotificationLevel(displayLevel);
if (decode) {
if (outFileName)
if (filenameIdx==1)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
else
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName);
} else { /* compression */
if (outFileName)
if (filenameIdx==1)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, ZSTD_EXTENSION, dictFileName, cLevel);
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, forceStdout ? NULL : ZSTD_EXTENSION, dictFileName, cLevel);
}
_end: