lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-01 09:47:01 +03:00
Code Activity

Added : Frame concatenation ability

Browse Source
This commit is contained in:
Yann Collet
2015-09-10 23:26:09 +01:00
parent 5abd8203cb
commit be50aaa0ee
5 changed files with 166 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
programs/Makefile
View File

@ -120,10 +120,22 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32
test-all: test test32 memtest
test-zstd: zstd datagen
@echo "*** flush write error test ***"
@echo "\n**** frame concatenation **** "
@echo "hello " > hello.tmp
@echo "world!" > world.tmp
@cat hello.tmp world.tmp > helloworld.tmp
./zstd hello.tmp > hello.zstd
./zstd world.tmp > world.zstd
@cat hello.zstd world.zstd > helloworld.zstd
./zstd -d helloworld.zstd > result.tmp
cat result.tmp
sdiff helloworld.tmp result.tmp
@rm *.tmp *.zstd
@echo frame concatenation test completed
@echo "**** flush write error test **** "
echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
@echo "*** zstd round-trip tests *** "
@echo "**** zstd round-trip tests **** "
./datagen | ./zstd -v | ./zstd -d > $(VOID)
./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID)
./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)

100
programs/fileio.c
View File

@ -309,45 +309,18 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
#define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
BYTE* inBuff, size_t inBuffSize,
BYTE* outBuff, size_t outBuffSize,
ZSTD_Dctx* dctx)
{
FILE* finput, *foutput;
BYTE* inBuff;
size_t inBuffSize;
BYTE* outBuff, *op, *oend;
size_t outBuffSize;
U32 blockSize = 128 KB;
U32 wNbBlocks = 4;
BYTE* op = outBuff;
BYTE* const oend = outBuff + outBuffSize;
U64 filesize = 0;
BYTE* header[MAXHEADERSIZE];
ZSTD_Dctx* dctx;
size_t toRead;
size_t sizeCheck;
/* Init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
dctx = ZSTD_createDCtx();
/* check header */
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(header, (size_t)1, toRead, finput);
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
/* Here later : blockSize determination */
/* Allocate Memory */
inBuffSize = blockSize + FIO_blockHeaderSize;
inBuff = (BYTE*)malloc(inBuffSize);
outBuffSize = wNbBlocks * blockSize;
outBuff = (BYTE*)malloc(outBuffSize);
op = outBuff;
oend = outBuff + outBuffSize;
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
/* Main decompression Loop */
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
while (toRead)
@ -380,6 +353,67 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
}
return filesize;
}
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
{
FILE* finput, *foutput;
BYTE* inBuff=NULL;
size_t inBuffSize = 0;
BYTE* outBuff=NULL;
size_t outBuffSize = 0;
U32 blockSize = 128 KB;
U32 wNbBlocks = 4;
U64 filesize = 0;
BYTE* header[MAXHEADERSIZE];
ZSTD_Dctx* dctx;
size_t toRead;
size_t sizeCheck;
/* Init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
dctx = ZSTD_createDCtx();
/* for each frame */
for ( ; ; )
{
/* check header */
ZSTD_resetDCtx(dctx);
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(header, (size_t)1, toRead, finput);
if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
/* Here later : blockSize determination */
/* Allocate Memory (if needed) */
{
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
size_t newOutBuffSize = wNbBlocks * blockSize;
if (newInBuffSize > inBuffSize)
{
free(inBuff);
inBuffSize = newInBuffSize;
inBuff = (BYTE*)malloc(inBuffSize);
}
if (newOutBuffSize > outBuffSize)
{
free(outBuff);
outBuffSize = newOutBuffSize;
outBuff = (BYTE*)malloc(outBuffSize);
}
}
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
}
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2,"Decoded %llu bytes \n", (long long unsigned)filesize);