lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-11-02 09:13:20 +03:00
Code Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into Other

Browse Source
This commit is contained in:
inikep
2016-08-17 14:02:36 +02:00
parent e4aef6c342 c411902230
commit bbf3739ac7
14 changed files with 1669 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
NEWS
View File

@@ -1,8 +1,10 @@
v0.8.1
New streaming API
Changed : --ultra now enables levels beyond 19
Changed : -i# now selects benchmark time in second
Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
Fixed : speed regression on specific patterns (#272)
Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
v0.8.0
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers

2
examples/.gitignore vendored
View File

@@ -3,6 +3,8 @@ simple_compression
simple_decompression
dictionary_compression
dictionary_decompression
streaming_compression
streaming_decompression
#test artefact
tmp*

21
examples/Makefile
View File

@@ -31,7 +31,8 @@ LDFLAGS+= -lzstd
default: all
all: simple_compression simple_decompression \
dictionary_compression dictionary_decompression
dictionary_compression dictionary_decompression \
streaming_compression streaming_decompression
simple_compression : simple_compression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -45,19 +46,29 @@ dictionary_compression : dictionary_compression.c
dictionary_decompression : dictionary_decompression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
streaming_compression : streaming_compression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
streaming_decompression : streaming_decompression.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
clean:
@rm -f core *.o tmp* result* *.zst \
simple_compression simple_decompression \
dictionary_compression dictionary_decompression
dictionary_compression dictionary_decompression \
streaming_compression streaming_decompression
@echo Cleaning completed
test: all
cp README.md tmp
@echo starting simple compression
./simple_compression tmp
@echo starting simple_decompression
./simple_decompression tmp.zst
@echo dictionary compression
./streaming_decompression tmp.zst
@echo starting streaming compression
./streaming_compression tmp
./streaming_decompression tmp.zst
@echo starting dictionary compression
./dictionary_compression tmp README.md
@echo dictionary decompression
./dictionary_decompression tmp.zst README.md
@echo tests completed

143
examples/streaming_compression.c Normal file
View File

@@ -0,0 +1,143 @@
/*
Streaming compression
Educational program using zstd library
Copyright (C) Yann Collet 2016
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- zstd homepage : http://www.zstd.net/
*/
#include <stdlib.h> // malloc, exit
#include <stdio.h> // fprintf, perror, feof
#include <string.h> // strerror
#include <errno.h> // errno
#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being
#include <zstd.h> // presumes zstd library is installed
static void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
if (buff) return buff;
/* error */
perror("malloc:");
exit(1);
}
static FILE* fopen_orDie(const char *filename, const char *instruction)
{
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
/* error */
perror(filename);
exit(3);
}
static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
{
size_t const readSize = fread(buffer, 1, sizeToRead, file);
if (readSize == sizeToRead) return readSize; /* good */
if (feof(file)) return readSize; /* good, reached end of file */
/* error */
perror("fread");
exit(4);
}
static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
{
size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
if (writtenSize == sizeToWrite) return sizeToWrite; /* good */
/* error */
perror("fwrite");
exit(5);
}
static size_t fclose_orDie(FILE* file)
{
if (!fclose(file)) return 0;
/* error */
perror("fclose");
exit(6);
}
static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
{
FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb");
size_t const buffInSize = ZSTD_CStreamInSize();;
void* const buffIn = malloc_orDie(buffInSize);
size_t const buffOutSize = ZSTD_CStreamOutSize();;
void* const buffOut = malloc_orDie(buffOutSize);
size_t read, toRead = buffInSize;
ZSTD_CStream* const cstream = ZSTD_createCStream();
if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
size_t const initResult = ZSTD_initCStream(cstream, cLevel);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); }
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
ZSTD_inBuffer input = { buffIn, read, 0 };
while (input.pos < input.size) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
toRead = ZSTD_compressStream(cstream, &output , &input);
fwrite_orDie(buffOut, output.pos, fout);
}
}
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
size_t const remainingToFlush = ZSTD_endStream(cstream, &output);
if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); }
fwrite_orDie(buffOut, output.pos, fout);
fclose_orDie(fout);
fclose_orDie(fin);
free(buffIn);
free(buffOut);
}
static const char* createOutFilename_orDie(const char* filename)
{
size_t const inL = strlen(filename);
size_t const outL = inL + 5;
void* outSpace = malloc_orDie(outL);
memset(outSpace, 0, outL);
strcat(outSpace, filename);
strcat(outSpace, ".zst");
return (const char*)outSpace;
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
const char* const inFilename = argv[1];
if (argc!=2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE\n", exeName);
return 1;
}
const char* const outFilename = createOutFilename_orDie(inFilename);
compressFile_orDie(inFilename, outFilename, 1);
return 0;
}

117
examples/streaming_decompression.c Normal file
View File

@@ -0,0 +1,117 @@
/*
Streaming compression
Educational program using zstd library
Copyright (C) Yann Collet 2016
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- zstd homepage : http://www.zstd.net/
*/
#include <stdlib.h> // malloc, exit
#include <stdio.h> // fprintf, perror, feof
#include <string.h> // strerror
#include <errno.h> // errno
#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being
#include <zstd.h> // presumes zstd library is installed
static void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
if (buff) return buff;
/* error */
perror("malloc:");
exit(1);
}
static FILE* fopen_orDie(const char *filename, const char *instruction)
{
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
/* error */
perror(filename);
exit(3);
}
static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
{
size_t const readSize = fread(buffer, 1, sizeToRead, file);
if (readSize == sizeToRead) return readSize; /* good */
if (feof(file)) return readSize; /* good, reached end of file */
/* error */
perror("fread");
exit(4);
}
static size_t fclose_orDie(FILE* file)
{
if (!fclose(file)) return 0;
/* error */
perror("fclose");
exit(6);
}
static void decompressFile_orDie(const char* fname)
{
FILE* const fin = fopen_orDie(fname, "rb");
size_t const buffInSize = ZSTD_DStreamInSize();;
void* const buffIn = malloc_orDie(buffInSize);
size_t const buffOutSize = ZSTD_DStreamOutSize();;
void* const buffOut = malloc_orDie(buffOutSize);
size_t read, toRead = buffInSize;
ZSTD_DStream* const dstream = ZSTD_createDStream();
if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
size_t const initResult = ZSTD_initDStream(dstream);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error \n"); exit(11); }
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
ZSTD_inBuffer input = { buffIn, read, 0 };
while (input.pos < input.size) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
toRead = ZSTD_decompressStream(dstream, &output , &input);
/* note : data is just "sinked" into buffOut
a more complete example would write it to disk or stdout */
}
}
fclose_orDie(fin);
free(buffIn);
free(buffOut);
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
const char* const inFilename = argv[1];
if (argc!=2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE\n", exeName);
return 1;
}
decompressFile_orDie(inFilename);
printf("%s correctly decoded (in memory). \n", inFilename);
return 0;
}

14
lib/README.md
View File

@@ -37,15 +37,18 @@ Other optional functionalities provided are :
- `legacy/` : source code to decompress previous versions of zstd, starting from `v0.1`.
This module also depends on `common/` and `decompress/` .
Note that it's required to compile the library with `ZSTD_LEGACY_SUPPORT = 1` .
Library compilation must include directive `ZSTD_LEGACY_SUPPORT = 1` .
The main API can be consulted in `legacy/zstd_legacy.h`.
Advanced API from each version can be found in its relevant header file.
For example, advanced API for version `v0.4` is in `zstd_v04.h` .
Advanced API from each version can be found in their relevant header file.
For example, advanced API for version `v0.4` is in `legacy/zstd_v04.h` .
#### Streaming API
#### Obsolete streaming API
Streaming is currently provided by `common/zbuff.h`.
Streaming is now provided within `zstd.h`.
Older streaming API is still provided within `common/zbuff.h`.
It is considered obsolete, and will be removed in a future version.
Consider migrating towards newer streaming API.
#### Miscellaneous
@@ -55,3 +58,4 @@ The other files are not source code. There are :
- LICENSE : contains the BSD license text
- Makefile : script to compile or install zstd library (static and dynamic)
- libzstd.pc.in : for pkg-config (`make install`)
- README.md : this file

273
lib/compress/zstd_compress.c
View File

@@ -32,7 +32,7 @@
*/
/* *******************************************************
/*-*******************************************************
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
@@ -55,7 +55,7 @@
#include "mem.h"
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH_reset, update, digest */
#define FSE_STATIC_LINKING_ONLY
#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
@@ -2758,6 +2758,275 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
/* ******************************************************************
* Streaming
********************************************************************/
typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
struct ZSTD_CStream_s {
ZSTD_CCtx* zc;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
size_t inBuffPos;
size_t inBuffTarget;
size_t blockSize;
char* outBuff;
size_t outBuffSize;
size_t outBuffContentSize;
size_t outBuffFlushedSize;
ZSTD_cStreamStage stage;
U32 checksum;
U32 frameEnded;
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CStream within "zstd.h" */
ZSTD_CStream* ZSTD_createCStream(void)
{
return ZSTD_createCStream_advanced(defaultCustomMem);
}
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
{
ZSTD_CStream* zcs;
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zcs = (ZSTD_CStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CStream));
if (zcs==NULL) return NULL;
memset(zcs, 0, sizeof(ZSTD_CStream));
memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
zcs->zc = ZSTD_createCCtx_advanced(customMem);
if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; }
return zcs;
}
size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
{
if (zcs==NULL) return 0; /* support free on NULL */
ZSTD_freeCCtx(zcs->zc);
if (zcs->inBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff);
if (zcs->outBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff);
zcs->customMem.customFree(zcs->customMem.opaque, zcs);
return 0;
}
/*====== Initialization ======*/
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
/* allocate buffers */
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
if (zcs->inBuffSize < neededInBuffSize) {
zcs->inBuffSize = neededInBuffSize;
zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff); /* should not be necessary */
zcs->inBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, neededInBuffSize);
if (zcs->inBuff == NULL) return ERROR(memory_allocation);
}
zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
}
if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) {
zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1;
zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff); /* should not be necessary */
zcs->outBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, zcs->outBuffSize);
if (zcs->outBuff == NULL) return ERROR(memory_allocation);
}
{ size_t const errorCode = ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize);
if (ZSTD_isError(errorCode)) return errorCode; }
zcs->inToCompress = 0;
zcs->inBuffPos = 0;
zcs->inBuffTarget = zcs->blockSize;
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load;
zcs->checksum = params.fParams.checksumFlag > 0;
zcs->frameEnded = 0;
return 0; /* ready to go */
}
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0);
}
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
{
return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
}
/*====== Compression ======*/
typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e;
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr,
ZSTD_flush_e const flush)
{
U32 someMoreWork = 1;
const char* const istart = (const char*)src;
const char* const iend = istart + *srcSizePtr;
const char* ip = istart;
char* const ostart = (char*)dst;
char* const oend = ostart + *dstCapacityPtr;
char* op = ostart;
while (someMoreWork) {
switch(zcs->stage)
{
case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
case zcss_load:
/* complete inBuffer */
{ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip);
zcs->inBuffPos += loaded;
ip += loaded;
if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) {
someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */
} }
/* compress current block (note : this stage cannot be stopped in the middle) */
{ void* cDst;
size_t cSize;
size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
size_t oSize = oend-op;
if (oSize >= ZSTD_compressBound(iSize))
cDst = op; /* compress directly into output buffer (avoid flush stage) */
else
cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = (flush == zsf_end) ?
ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize;
if (flush == zsf_end) zcs->frameEnded = 1;
/* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
if (zcs->inBuffTarget > zcs->inBuffSize)
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */
zcs->inToCompress = zcs->inBuffPos;
if (cDst == op) { op += cSize; break; } /* no need to flush */
zcs->outBuffContentSize = cSize;
zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_flush; /* pass-through to flush stage */
}
case zcss_flush:
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed;
zcs->outBuffFlushedSize += flushed;
if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load;
break;
}
case zcss_final:
someMoreWork = 0; /* do nothing */
break;
default:
return ERROR(GENERIC); /* impossible */
}
}
*srcSizePtr = ip - istart;
*dstCapacityPtr = op - ostart;
if (zcs->frameEnded) return 0;
{ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
if (hintInSize==0) hintInSize = zcs->blockSize;
return hintInSize;
}
}
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
size_t sizeRead = input->size - input->pos;
size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs,
(char*)(output->dst) + output->pos, &sizeWritten,
(const char*)(input->src) + input->pos, &sizeRead, zsf_gather);
input->pos += sizeRead;
output->pos += sizeWritten;
return result;
}
/*====== Finalize ======*/
/*! ZSTD_flushStream() :
* @return : amount of data remaining to flush */
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{
size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs,
(char*)(output->dst) + output->pos, &sizeWritten,
&srcSize, &srcSize, /* use a valid src address instead of NULL */
zsf_flush);
output->pos += sizeWritten;
if (ZSTD_isError(result)) return result;
return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
}
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{
BYTE* const ostart = (BYTE*)(output->dst) + output->pos;
BYTE* const oend = (BYTE*)(output->dst) + output->size;
BYTE* op = ostart;
if (zcs->stage != zcss_final) {
/* flush whatever remains */
size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos;
size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */
size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
op += sizeWritten;
if (remainingToFlush) {
output->pos += sizeWritten;
return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4);
}
/* create epilogue */
zcs->stage = zcss_final;
zcs->outBuffContentSize = !notEnded ? 0 :
ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */
}
/* flush epilogue */
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed;
zcs->outBuffFlushedSize += flushed;
output->pos += op-ostart;
if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */
return toFlush - flushed;
}
}
/*-===== Pre-defined compression levels =====-*/
#define ZSTD_DEFAULT_CLEVEL 1

236
lib/decompress/zstd_decompress.c
View File

@@ -1301,3 +1301,239 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
dst, dstCapacity,
src, srcSize);
}
/*=====================================
* Streaming decompression
*====================================*/
typedef enum { zdss_init, zdss_loadHeader,
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
/* *** Resource management *** */
struct ZSTD_DStream_s {
ZSTD_DCtx* zd;
ZSTD_frameParams fParams;
ZSTD_dStreamStage stage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t lhSize;
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_DStream within "zstd.h" */
ZSTD_DStream* ZSTD_createDStream(void)
{
return ZSTD_createDStream_advanced(defaultCustomMem);
}
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
{
ZSTD_DStream* zds;
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zds = (ZSTD_DStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DStream));
if (zds==NULL) return NULL;
memset(zds, 0, sizeof(ZSTD_DStream));
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
zds->zd = ZSTD_createDCtx_advanced(customMem);
if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; }
zds->stage = zdss_init;
return zds;
}
size_t ZSTD_freeDStream(ZSTD_DStream* zds)
{
if (zds==NULL) return 0; /* support free on null */
ZSTD_freeDCtx(zds->zd);
if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
zds->customMem.customFree(zds->customMem.opaque, zds);
return 0;
}
/* *** Initialization *** */
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
{
zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize);
}
size_t ZSTD_initDStream(ZSTD_DStream* zds)
{
return ZSTD_initDStream_usingDict(zds, NULL, 0);
}
/* *** Decompression *** */
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
const char* const istart = (const char*)(input->src) + input->pos;
const char* const iend = (const char*)(input->src) + input->size;
const char* ip = istart;
char* const ostart = (char*)(output->dst) + output->pos;
char* const oend = (char*)(output->dst) + output->size;
char* op = ostart;
U32 someMoreWork = 1;
while (someMoreWork) {
switch(zds->stage)
{
case zdss_init :
return ERROR(init_missing);
case zdss_loadHeader :
{ size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize);
if (ZSTD_isError(hSize)) return hSize;
if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
zds->lhSize += iend-ip;
input->pos = input->size;
return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
break;
} }
/* Consume header */
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */
size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size);
if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */
if (h1Size < zds->lhSize) { /* long header */
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd);
size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size);
if (ZSTD_isError(h2Result)) return h2Result;
} }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
/* Frame header instruct buffer sizes */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize;
zds->blockSize = blockSize;
if (zds->inBuffSize < blockSize) {
zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
zds->inBuffSize = blockSize;
zds->inBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, blockSize);
if (zds->inBuff == NULL) return ERROR(memory_allocation);
}
if (zds->outBuffSize < neededOutSize) {
zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
zds->outBuffSize = neededOutSize;
zds->outBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, neededOutSize);
if (zds->outBuff == NULL) return ERROR(memory_allocation);
} }
zds->stage = zdss_read;
/* pass-through */
case zdss_read:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
if (neededInSize==0) { /* end of frame */
zds->stage = zdss_init;
someMoreWork = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush;
break;
}
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
zds->stage = zdss_load;
/* pass-through */
}
case zdss_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
ip += loadedSize;
zds->inPos += loadedSize;
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
zds->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
zds->inPos = 0; /* input is consumed */
if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */
zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush;
/* pass-through */
} }
case zdss_flush:
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
op += flushedSize;
zds->outStart += flushedSize;
if (flushedSize == toFlushSize) { /* flush completed */
zds->stage = zdss_read;
if (zds->outStart + zds->blockSize > zds->outBuffSize)
zds->outStart = zds->outEnd = 0;
break;
}
/* cannot flush everything */
someMoreWork = 0;
break;
}
default: return ERROR(GENERIC); /* impossible */
} }
/* result */
input->pos += (size_t)(ip-istart);
output->pos += (size_t)(op-ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/
return nextSrcSizeHint;
}
}

160
lib/zstd.h
View File

@@ -36,7 +36,7 @@
extern "C" {
#endif
/*====== Dependency ======*/
/*====== Dependency ======*/
#include <stddef.h> /* size_t */
@@ -52,7 +52,7 @@ extern "C" {
#endif
/*====== Version ======*/
/*======= Version =======*/
#define ZSTD_VERSION_MAJOR 0
#define ZSTD_VERSION_MINOR 8
#define ZSTD_VERSION_RELEASE 1
@@ -84,15 +84,13 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 2 : decompressed size is an optional field, that may not be present.
* When `return==0`, consider data to decompress could have any size.
* In which case, it's necessary to use streaming mode to decompress data,
* or rely on application's implied limits.
* (For example, it may know that its own data is necessarily cut into blocks <= 16 KB).
* When `return==0`, data to decompress can have any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application may rely on its own implied limits.
* (For example, application own data could be necessarily cut into blocks <= 16 KB).
* note 3 : decompressed size could be wrong or intentionally modified !
* Always ensure result fits within application's authorized limits !
* Each application can have its own set of conditions.
* If the intention is to decompress public data compressed by zstd command line utility,
* it is recommended to support at least 8 MB for extended compatibility.
* Each application can set its own limits.
* note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
@@ -100,7 +98,7 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
`compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail.
`dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ).
If originalSize is unknown, and if there is no implied application-specific limitations,
it's necessary to use streaming mode to decompress data.
it's preferable to use streaming mode to decompress data.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
@@ -141,7 +139,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
***************************/
/*! ZSTD_compress_usingDict() :
* Compression using a predefined Dictionary (see dictBuilder/zdict.h).
* Note : This function load the dictionary, resulting in a significant startup time. */
* Note : This function load the dictionary, resulting in significant startup delay. */
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -151,7 +149,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
/*! ZSTD_decompress_usingDict() :
* Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
* Dictionary must be identical to the one used during compression.
* Note : This function load the dictionary, resulting in a significant startup time */
* Note : This function load the dictionary, resulting in significant startup delay */
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -163,7 +161,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
****************************/
/*! ZSTD_createCDict() :
* Create a digested dictionary, ready to start compression operation without startup delay.
* `dict` can be released after creation */
* `dict` can be released after ZSTD_CDict creation */
typedef struct ZSTD_CDict_s ZSTD_CDict;
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -232,19 +230,19 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */
typedef struct {
unsigned windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */
unsigned chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
unsigned hashLog; /*< dispatch table : larger == faster, more memory */
unsigned searchLog; /*< nb of searches : larger == more compression, slower */
unsigned searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */
unsigned targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
unsigned hashLog; /**< dispatch table : larger == faster, more memory */
unsigned searchLog; /**< nb of searches : larger == more compression, slower */
unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */
unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
ZSTD_strategy strategy;
} ZSTD_compressionParameters;
typedef struct {
unsigned contentSizeFlag; /*< 1: content size will be in frame header (if known). */
unsigned checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
unsigned noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */
unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
} ZSTD_frameParameters;
typedef struct {
@@ -323,12 +321,122 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx);
/* ******************************************************************
* Buffer-less streaming functions (synchronous mode)
* Streaming
********************************************************************/
typedef struct ZSTD_inBuffer_s {
const void* src; /**< start of input buffer */
size_t size; /**< size of input buffer */
size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_inBuffer;
typedef struct ZSTD_outBuffer_s {
void* dst; /**< start of output buffer */
size_t size; /**< size of output buffer */
size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_outBuffer;
/*====== compression ======*/
/*-***********************************************************************
* Streaming compression - howto
*
* A ZSTD_CStream object is required to track streaming operation.
* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
*
* Start by initializing ZSTD_CStream.
* Use ZSTD_initCStream() to start a new compression operation.
* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
*
* Use ZSTD_compressStream() repetitively to consume input stream.
* The function will automatically update both `pos`.
* Note that it may not consume the entire input, in which case `pos < size`,
* and it's up to the caller to present again remaining data.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
* or an error code, which can be tested using ZSTD_isError().
*
* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
* `output->pos` will be updated.
* Note some content might still be left within internal buffer if `output->size` is too small.
* @return : nb of bytes still present within internal buffer (0 if it's empty)
* or an error code, which can be tested using ZSTD_isError().
*
* ZSTD_endStream() instructs to finish a frame.
* It will perform a flush and write frame epilogue.
* The epilogue is required for decoders to consider a frame completed.
* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
* In which case, call again ZSTD_endStream() to complete the flush.
* @return : nb of bytes still present within internal buffer (0 if it's empty)
* or an error code, which can be tested using ZSTD_isError().
*
* *******************************************************************/
typedef struct ZSTD_CStream_s ZSTD_CStream;
ZSTD_CStream* ZSTD_createCStream(void);
size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
/* advanced */
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
/*====== decompression ======*/
/*-***************************************************************************
* Streaming decompression howto
*
* A ZSTD_DStream object is required to track streaming operations.
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
* ZSTD_DStream objects can be re-init multiple times.
*
* Use ZSTD_initDStream() to start a new decompression operation,
* or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
*
* Use ZSTD_decompressStream() repetitively to consume your input.
* The function will update both `pos`.
* Note that it may not consume the entire input (pos < size),
* in which case it's up to the caller to present remaining input again.
* @return : 0 when a frame is completely decoded and fully flushed,
* 1 when there is still some data left within internal buffer to flush,
* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted),
* or an error code, which can be tested using ZSTD_isError().
*
* *******************************************************************************/
typedef struct ZSTD_DStream_s ZSTD_DStream;
ZSTD_DStream* ZSTD_createDStream(void);
size_t ZSTD_freeDStream(ZSTD_DStream* zds);
size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */
size_t ZSTD_initDStream(ZSTD_DStream* zds);
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
/* advanced */
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
/* ******************************************************************
* Buffer-less and synchronous inner streaming functions
********************************************************************/
/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
* But it's also a complex one, with a lot of restrictions (documented below).
* For an easier streaming API, look into common/zbuff.h
* which removes all restrictions by allocating and managing its own internal buffer */
* But it's also a complex one, with many restrictions (documented below).
* Prefer using normal streaming API for an easier experience */
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);

2
programs/.gitignore vendored
View File

@@ -9,6 +9,8 @@ fuzzer
fuzzer32
zbufftest
zbufftest32
zstreamtest
zstreamtest32
datagen
paramgrill
paramgrill32

26
programs/Makefile
View File

@@ -94,7 +94,7 @@ all32: cleano32 all
$(ZSTDDIR)/decompress/zstd_decompress.o: CFLAGS += $(ALIGN_LOOP)
zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
@@ -116,15 +116,15 @@ zstd-pgo : clean zstd
rm zstd
$(MAKE) zstd MOREFLAGS=-fprofile-use
zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c
zstd-frugal: $(ZSTD_FILES) zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT)
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
$(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
$(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
zstd-small: clean
@@ -152,6 +152,13 @@ zbufftest32 : CFLAGS += -m32
zbufftest32 : EXT := 32$(EXT)
zbufftest32 : zbufftest
zstreamtest : $(ZSTD_FILES) datagen.c zstreamtest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstreamtest32 : CFLAGS += -m32
zstreamtest32 : EXT := 32$(EXT)
zstreamtest32 : zstreamtest
paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
@@ -229,9 +236,9 @@ ifneq (,$(filter $(HOST_OS),MSYS POSIX))
zstd-playTests: datagen
ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zbuff
test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32
test-all: test test32 valgrindTest
@@ -263,4 +270,11 @@ test-zbuff: zbufftest
test-zbuff32: zbufftest32
./zbufftest32 $(ZBUFFTEST)
test-zstream: zstreamtest
./zstreamtest $(ZBUFFTEST)
test-zstream32: zstreamtest32
./zstreamtest32 $(ZBUFFTEST)
endif

91
programs/fileio.c
View File

@@ -22,8 +22,7 @@
- zstd homepage : http://www.zstd.net
*/
/*
Note : this is stand-alone program.
It is not part of ZSTD compression library, it is a user program of ZSTD library.
Note : this file is part of zstd command line, which is not library.
The license of ZSTD library is BSD.
The license of this file is GPLv2.
*/
@@ -41,8 +40,11 @@
/* *************************************
* Compiler Options
***************************************/
#define _POSIX_SOURCE 1 /* enable %llu on Windows */
#define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */
#ifdef _MSC_VER /* Visual */
# define _POSIX_SOURCE 1 /* enable %llu on Windows */
# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */
# pragma warning(disable : 4204) /* non-constant aggregate initializer */
#endif
/*-*************************************
@@ -59,8 +61,6 @@
#include "fileio.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "zstd.h"
#define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_legacy.h" /* ZSTD_isLegacy */
@@ -173,10 +173,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
SET_BINARY_MODE(stdin);
} else {
f = fopen(srcFileName, "rb");
if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
}
if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return f;
}
@@ -212,9 +211,9 @@ static FILE* FIO_openDstFile(const char* dstFileName)
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */
} } }
f = fopen( dstFileName, "wb" );
if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
}
if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
return f;
}
@@ -265,7 +264,7 @@ typedef struct {
size_t dstBufferSize;
void* dictBuffer;
size_t dictBufferSize;
ZBUFF_CCtx* ctx;
ZSTD_CStream* cctx;
FILE* dstFile;
FILE* srcFile;
} cRess_t;
@@ -275,11 +274,11 @@ static cRess_t FIO_createCResources(const char* dictFileName)
cRess_t ress;
memset(&ress, 0, sizeof(ress));
ress.ctx = ZBUFF_createCCtx();
if (ress.ctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZBUFF context");
ress.srcBufferSize = ZBUFF_recommendedCInSize();
ress.cctx = ZSTD_createCStream();
if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream");
ress.srcBufferSize = ZSTD_CStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZBUFF_recommendedCOutSize();
ress.dstBufferSize = ZSTD_CStreamOutSize();
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory");
@@ -295,8 +294,8 @@ static void FIO_freeCResources(cRess_t ress)
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
errorCode = ZBUFF_freeCCtx(ress.ctx);
if (ZBUFF_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
errorCode = ZSTD_freeCStream(ress.cctx);
if (ZSTD_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZSTD_CStream : %s", ZSTD_getErrorName(errorCode));
}
@@ -324,8 +323,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
params.cParams.windowLog = g_maxWLog;
params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize);
}
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
{ size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZSTD_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(errorCode));
} }
/* Main compression loop */
@@ -337,30 +336,30 @@ static int FIO_compressFilename_internal(cRess_t ress,
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20));
/* Compress using buffered streaming */
{ size_t usedInSize = inSize;
size_t cSize = ress.dstBufferSize;
{ size_t const result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
if (ZBUFF_isError(result)) EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result)); }
if (inSize != usedInSize)
{ ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 };
ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
{ size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff);
if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result)); }
if (inBuff.pos != inBuff.size)
/* inBuff should be entirely consumed since buffer sizes are recommended ones */
EXM_THROW(24, "Compression error : input block not fully consumed");
/* Write cBlock */
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
compressedfilesize += cSize;
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
compressedfilesize += outBuff.pos;
}
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100);
}
/* End of Frame */
{ size_t cSize = ress.dstBufferSize;
size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
{ ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const result = ZSTD_endStream(ress.cctx, &outBuff);
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
compressedfilesize += cSize;
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
compressedfilesize += outBuff.pos;
}
/* Status */
@@ -496,7 +495,7 @@ typedef struct {
size_t dstBufferSize;
void* dictBuffer;
size_t dictBufferSize;
ZBUFF_DCtx* dctx;
ZSTD_DStream* dctx;
FILE* dstFile;
} dRess_t;
@@ -506,11 +505,11 @@ static dRess_t FIO_createDResources(const char* dictFileName)
memset(&ress, 0, sizeof(ress));
/* Allocation */
ress.dctx = ZBUFF_createDCtx();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
ress.srcBufferSize = ZBUFF_recommendedDInSize();
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
ress.srcBufferSize = ZSTD_DStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZBUFF_recommendedDOutSize();
ress.dstBufferSize = ZSTD_DStreamOutSize();
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
@@ -522,8 +521,8 @@ static dRess_t FIO_createDResources(const char* dictFileName)
static void FIO_freeDResources(dRess_t ress)
{
size_t const errorCode = ZBUFF_freeDCtx(ress.dctx);
if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
size_t const errorCode = ZSTD_freeDStream(ress.dctx);
if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", ZSTD_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
@@ -615,7 +614,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
size_t readSize;
U32 storedSkips = 0;
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
ZSTD_initDStream_usingDict(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
/* Header loading (optional, saves one loop) */
{ size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */
@@ -625,18 +624,18 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
/* Main decompression Loop */
while (1) {
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
readSize -= inSize;
ZSTD_inBuffer inBuff = { ress.srcBuffer, readSize, 0 };
ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const toRead = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead));
/* Write block */
storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, decodedSize, storedSkips);
frameSize += decodedSize;
storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips);
frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break; /* end of frame */
if (readSize) EXM_THROW(37, "Decoding error : should consume entire input");
if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
/* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(38, "too large block");

659
programs/zstreamtest.c Normal file
View File

@@ -0,0 +1,659 @@
/*
Fuzzer test tool for zstd streaming API
Copyright (C) Yann Collet 2016
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- ZSTD homepage : https://www.zstd.net/
*/
/*-************************************
* Compiler specific
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS /* fgets */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
#endif
/*-************************************
* Includes
**************************************/
#include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */
#include <sys/timeb.h> /* timeb */
#include <string.h> /* strcmp */
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
#include "zstd.h" /* ZSTD_compressBound */
#include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64_* */
/*-************************************
* Constants
**************************************/
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
static const U32 nbTestsDefault = 10000;
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
/*-************************************
* Display Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const U32 g_refreshRate = 150;
static U32 g_displayTime = 0;
static U32 g_testTime = 0;
/*-*******************************************************
* Fuzzer functions
*********************************************************/
#define MAX(a,b) ((a)>(b)?(a):(b))
static U32 FUZ_GetMilliStart(void)
{
struct timeb tb;
U32 nCount;
ftime( &tb );
nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm);
return nCount;
}
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
{
U32 const nCurrent = FUZ_GetMilliStart();
U32 nSpan = nCurrent - nTimeStart;
if (nTimeStart > nCurrent)
nSpan += 0x100000 * 1000;
return nSpan;
}
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* seedPtr)
{
U32 rand32 = *seedPtr;
rand32 *= prime1;
rand32 += prime2;
rand32 = FUZ_rotl32(rand32, 13);
*seedPtr = rand32;
return rand32 >> 5;
}
/*
static unsigned FUZ_highbit32(U32 v32)
{
unsigned nbBits = 0;
if (v32==0) return 0;
for ( ; v32 ; v32>>=1) nbBits++;
return nbBits;
}
*/
static void* allocFunction(void* opaque, size_t size)
{
void* address = malloc(size);
(void)opaque;
return address;
}
static void freeFunction(void* opaque, void* address)
{
(void)opaque;
free(address);
}
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
{
int testResult = 0;
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* CNBuffer = malloc(CNBufferSize);
size_t const skippableFrameSize = 11;
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize);
size_t cSize;
U32 testNb=0;
ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
ZSTD_inBuffer inBuff;
ZSTD_outBuffer outBuff;
/* Create compressible test buffer */
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
DISPLAY("Not enough memory, aborting\n");
goto _output_error;
}
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
/* generate skippable frame */
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
cSize = skippableFrameSize + 8;
/* Basic compression test */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1);
outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
{ size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /*< error, or some data not flushed */
cSize += outBuff.pos;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
/* skippable frame test */
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
outBuff.dst = decodedBuffer;
outBuff.size = CNBufferSize;
outBuff.pos = 0;
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; }
if (outBuff.pos != 0) goto _output_error; /* skippable frame len is 0 */
DISPLAYLEVEL(4, "OK \n");
/* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t i;
for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
/* Byte-by-byte decompression test */
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
{ size_t r = 1;
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
inBuff.src = compressedBuffer;
outBuff.dst = decodedBuffer;
inBuff.pos = 0;
outBuff.pos = 0;
while (r) { /* skippable frame */
inBuff.size = inBuff.pos + 1;
outBuff.size = outBuff.pos + 1;
r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error;
}
ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
r=1;
while (r) { /* normal frame */
inBuff.size = inBuff.pos + 1;
outBuff.size = outBuff.pos + 1;
r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(r)) goto _output_error;
}
}
if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t i;
for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n");
_end:
ZSTD_freeCStream(zc);
ZSTD_freeDStream(zd);
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
{
const BYTE* b1 = (const BYTE*)buf1;
const BYTE* b2 = (const BYTE*)buf2;
size_t u;
for (u=0; u<max; u++) {
if (b1[u] != b2[u]) break;
}
return u;
}
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
{
size_t const lengthMask = ((size_t)1 << logLength) - 1;
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
}
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
{
U32 const logLength = FUZ_rand(seed) % maxLog;
return FUZ_rLogLength(seed, logLength);
}
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
BYTE* cNoiseBuffer[5];
size_t srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* copyBuffer;
size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
BYTE* cBuffer;
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* dstBuffer;
size_t dstBufferSize = srcBufferSize;
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed;
ZSTD_CStream* zc;
ZSTD_DStream* zd;
U32 startTime = FUZ_GetMilliStart();
/* allocations */
zc = ZSTD_createCStream();
zd = ZSTD_createDStream();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
copyBuffer= (BYTE*)malloc (copyBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
const BYTE* dict;
size_t maxTestSize, dictSize;
size_t cSize, totalTestSize, totalGenSize;
U32 n, nbChunks;
XXH64_state_t xxhState;
U64 crcOrig;
/* init */
DISPLAYUPDATE(2, "\r%6u", testNb);
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
/* states full reset (unsynchronized) */
/* some issues only happen when reusing states in a specific sequence of parameters */
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); }
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); }
/* srcBuffer selection [0-4] */
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
else {
buffNb >>= 3;
if (buffNb & 7) {
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
buffNb = tnb[buffNb >> 3];
} else {
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
buffNb = tnb[buffNb >> 3];
} }
srcBuffer = cNoiseBuffer[buffNb];
}
/* compression init */
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
/* random dictionary selection */
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
{ size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0);
CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError));
} } }
/* multi-segments compression test */
XXH64_reset(&xxhState, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
{ ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
/* compress random chunk into random size dst buffer */
{ size_t const readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, readChunkSize, 0 };
outBuff.size = outBuff.pos + dstBuffSize;
{ size_t const compressionError = ZSTD_compressStream(zc, &outBuff, &inBuff);
CHECK (ZSTD_isError(compressionError), "compression error : %s", ZSTD_getErrorName(compressionError)); }
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
totalTestSize += inBuff.pos;
}
/* random flush operation, to mess around */
if ((FUZ_rand(&lseed) & 15) == 0) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
outBuff.size = outBuff.pos + adjustedDstSize;
{ size_t const flushError = ZSTD_flushStream(zc, &outBuff);
CHECK (ZSTD_isError(flushError), "flush error : %s", ZSTD_getErrorName(flushError));
} } }
/* final frame epilogue */
{ size_t remainingToFlush = (size_t)(-1);
while (remainingToFlush) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
U32 const enoughDstSize = (adjustedDstSize >= remainingToFlush);
outBuff.size = outBuff.pos + adjustedDstSize;
remainingToFlush = ZSTD_endStream(zc, &outBuff);
CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush));
CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush);
} }
crcOrig = XXH64_digest(&xxhState);
cSize = outBuff.pos;
}
/* multi - fragments decompression test */
ZSTD_initDStream_usingDict(zd, dict, dictSize);
{ size_t decompressionResult = 1;
ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
for (totalGenSize = 0 ; decompressionResult ; ) {
size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
inBuff.size = inBuff.pos + readCSrcSize;
outBuff.size = inBuff.pos + dstBuffSize;
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
}
CHECK (decompressionResult != 0, "frame not fully decoded");
CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size")
CHECK (inBuff.pos != cSize, "compressed data should be fully read")
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
} }
/*===== noisy/erroneous src decompression test =====*/
/* add some noise */
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
} }
/* try decompression on noisy data */
ZSTD_initDStream(zd);
{ ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
while (outBuff.pos < dstBufferSize) {
size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
outBuff.size = outBuff.pos + adjustedDstSize;
inBuff.size = inBuff.pos + randomCSrcSize;
{ size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(decompressError)) break; /* error correctly detected */
} } } }
DISPLAY("\r%u fuzzer tests completed \n", testNb);
_cleanup:
ZSTD_freeCStream(zc);
ZSTD_freeDStream(zd);
free(cNoiseBuffer[0]);
free(cNoiseBuffer[1]);
free(cNoiseBuffer[2]);
free(cNoiseBuffer[3]);
free(cNoiseBuffer[4]);
free(copyBuffer);
free(cBuffer);
free(dstBuffer);
return result;
_output_error:
result = 1;
goto _cleanup;
}
/*-*******************************************************
* Command line
*********************************************************/
int FUZ_usage(const char* programName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
DISPLAY( " -v : verbose\n");
DISPLAY( " -p : pause at the end\n");
DISPLAY( " -h : display help and exit\n");
return 0;
}
int main(int argc, const char** argv)
{
U32 seed=0;
int seedset=0;
int argNb;
int nbTests = nbTestsDefault;
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0;
U32 mainPause = 0;
const char* programName = argv[0];
ZSTD_customMem customMem = { allocFunction, freeFunction, NULL };
ZSTD_customMem customNULL = { NULL, NULL, NULL };
/* Check command line */
for(argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
/* Parsing commands. Aggregated commands are allowed */
if (argument[0]=='-') {
argument++;
while (*argument!=0) {
switch(*argument)
{
case 'h':
return FUZ_usage(programName);
case 'v':
argument++;
g_displayLevel=4;
break;
case 'q':
argument++;
g_displayLevel--;
break;
case 'p': /* pause at the end */
argument++;
mainPause = 1;
break;
case 'i':
argument++;
nbTests=0; g_testTime=0;
while ((*argument>='0') && (*argument<='9')) {
nbTests *= 10;
nbTests += *argument - '0';
argument++;
}
break;
case 'T':
argument++;
nbTests=0; g_testTime=0;
while ((*argument>='0') && (*argument<='9')) {
g_testTime *= 10;
g_testTime += *argument - '0';
argument++;
}
if (*argument=='m') g_testTime *=60, argument++;
if (*argument=='n') argument++;
g_testTime *= 1000;
break;
case 's':
argument++;
seed=0;
seedset=1;
while ((*argument>='0') && (*argument<='9')) {
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 't':
argument++;
testNb=0;
while ((*argument>='0') && (*argument<='9')) {
testNb *= 10;
testNb += *argument - '0';
argument++;
}
break;
case 'P': /* compressibility % */
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9')) {
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
default:
return FUZ_usage(programName);
}
} } } /* for(argNb=1; argNb<argc; argNb++) */
/* Get Seed */
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
DISPLAY("Seed = %u\n", seed);
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
if (nbTests<=0) nbTests=1;
if (testNb==0) {
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
if (!result) {
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
} }
if (!result)
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
if (mainPause) {
int unused;
DISPLAY("Press Enter \n");
unused = getchar();
(void)unused;
}
return result;
}

14
zlibWrapper/zstd_zlibwrapper.c
View File

@@ -227,7 +227,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!");
if (flush == Z_FINISH || flush == Z_FULL_FLUSH) {
if (flush == Z_FINISH) {
size_t bytesLeft;
size_t dstCapacity = strm->avail_out;
if (zwc->bytesLeft) {
@@ -246,6 +246,18 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
if (flush == Z_FINISH && bytesLeft == 0) return Z_STREAM_END;
zwc->bytesLeft = bytesLeft;
}
if (flush == Z_SYNC_FLUSH) {
size_t bytesLeft;
size_t dstCapacity = strm->avail_out;
bytesLeft = ZBUFF_compressFlush(zwc->zbc, strm->next_out, &dstCapacity);
LOG_WRAPPER("ZBUFF_compressFlush avail_out=%d dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)dstCapacity, (int)bytesLeft);
if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR;
strm->next_out += dstCapacity;
strm->total_out += dstCapacity;
strm->avail_out -= dstCapacity;
zwc->bytesLeft = bytesLeft;
}
return Z_OK;
}