diff --git a/NEWS b/NEWS index 20765a3d7..36bd15242 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,10 @@ v0.8.1 +New streaming API Changed : --ultra now enables levels beyond 19 Changed : -i# now selects benchmark time in second Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell Fixed : speed regression on specific patterns (#272) +Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) v0.8.0 Improved : better speed on clang and gcc -O2, thanks to Eric Biggers diff --git a/examples/.gitignore b/examples/.gitignore index 9d241dba6..1c98e1884 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -3,6 +3,8 @@ simple_compression simple_decompression dictionary_compression dictionary_decompression +streaming_compression +streaming_decompression #test artefact tmp* diff --git a/examples/Makefile b/examples/Makefile index 6ba8655b4..a9d608778 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -31,7 +31,8 @@ LDFLAGS+= -lzstd default: all all: simple_compression simple_decompression \ - dictionary_compression dictionary_decompression + dictionary_compression dictionary_decompression \ + streaming_compression streaming_decompression simple_compression : simple_compression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ @@ -45,19 +46,29 @@ dictionary_compression : dictionary_compression.c dictionary_decompression : dictionary_decompression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +streaming_compression : streaming_compression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +streaming_decompression : streaming_decompression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + clean: @rm -f core *.o tmp* result* *.zst \ simple_compression simple_decompression \ - dictionary_compression dictionary_decompression + dictionary_compression dictionary_decompression \ + streaming_compression streaming_decompression @echo Cleaning completed test: all cp README.md tmp + @echo starting simple compression ./simple_compression tmp - @echo starting simple_decompression ./simple_decompression tmp.zst - @echo dictionary compression + ./streaming_decompression tmp.zst + @echo starting streaming compression + ./streaming_compression tmp + ./streaming_decompression tmp.zst + @echo starting dictionary compression ./dictionary_compression tmp README.md - @echo dictionary decompression ./dictionary_decompression tmp.zst README.md @echo tests completed diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c new file mode 100644 index 000000000..e2568836a --- /dev/null +++ b/examples/streaming_compression.c @@ -0,0 +1,143 @@ +/* + Streaming compression + Educational program using zstd library + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + +#include // malloc, exit +#include // fprintf, perror, feof +#include // strerror +#include // errno +#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being +#include // presumes zstd library is installed + + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc:"); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(5); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + + +static void compressFile_orDie(const char* fname, const char* outName, int cLevel) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + FILE* const fout = fopen_orDie(outName, "wb"); + size_t const buffInSize = ZSTD_CStreamInSize();; + void* const buffIn = malloc_orDie(buffInSize); + size_t const buffOutSize = ZSTD_CStreamOutSize();; + void* const buffOut = malloc_orDie(buffOutSize); + size_t read, toRead = buffInSize; + + ZSTD_CStream* const cstream = ZSTD_createCStream(); + if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } + size_t const initResult = ZSTD_initCStream(cstream, cLevel); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); } + + while( (read = fread_orDie(buffIn, toRead, fin)) ) { + ZSTD_inBuffer input = { buffIn, read, 0 }; + while (input.pos < input.size) { + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + toRead = ZSTD_compressStream(cstream, &output , &input); + fwrite_orDie(buffOut, output.pos, fout); + } + } + + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + size_t const remainingToFlush = ZSTD_endStream(cstream, &output); + if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); } + fwrite_orDie(buffOut, output.pos, fout); + + fclose_orDie(fout); + fclose_orDie(fin); + free(buffIn); + free(buffOut); +} + + +static const char* createOutFilename_orDie(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* outSpace = malloc_orDie(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (const char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + const char* const inFilename = argv[1]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + const char* const outFilename = createOutFilename_orDie(inFilename); + compressFile_orDie(inFilename, outFilename, 1); + + return 0; +} diff --git a/examples/streaming_decompression.c b/examples/streaming_decompression.c new file mode 100644 index 000000000..a9efd7a2d --- /dev/null +++ b/examples/streaming_decompression.c @@ -0,0 +1,117 @@ +/* + Streaming compression + Educational program using zstd library + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + +#include // malloc, exit +#include // fprintf, perror, feof +#include // strerror +#include // errno +#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being +#include // presumes zstd library is installed + + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc:"); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + + +static void decompressFile_orDie(const char* fname) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + size_t const buffInSize = ZSTD_DStreamInSize();; + void* const buffIn = malloc_orDie(buffInSize); + size_t const buffOutSize = ZSTD_DStreamOutSize();; + void* const buffOut = malloc_orDie(buffOutSize); + size_t read, toRead = buffInSize; + + ZSTD_DStream* const dstream = ZSTD_createDStream(); + if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); } + size_t const initResult = ZSTD_initDStream(dstream); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error \n"); exit(11); } + + while( (read = fread_orDie(buffIn, toRead, fin)) ) { + ZSTD_inBuffer input = { buffIn, read, 0 }; + while (input.pos < input.size) { + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + toRead = ZSTD_decompressStream(dstream, &output , &input); + /* note : data is just "sinked" into buffOut + a more complete example would write it to disk or stdout */ + } + } + + fclose_orDie(fin); + free(buffIn); + free(buffOut); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + const char* const inFilename = argv[1]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + decompressFile_orDie(inFilename); + printf("%s correctly decoded (in memory). \n", inFilename); + + return 0; +} diff --git a/lib/README.md b/lib/README.md index a3087f05e..f6dbc4181 100644 --- a/lib/README.md +++ b/lib/README.md @@ -37,15 +37,18 @@ Other optional functionalities provided are : - `legacy/` : source code to decompress previous versions of zstd, starting from `v0.1`. This module also depends on `common/` and `decompress/` . - Note that it's required to compile the library with `ZSTD_LEGACY_SUPPORT = 1` . + Library compilation must include directive `ZSTD_LEGACY_SUPPORT = 1` . The main API can be consulted in `legacy/zstd_legacy.h`. - Advanced API from each version can be found in its relevant header file. - For example, advanced API for version `v0.4` is in `zstd_v04.h` . + Advanced API from each version can be found in their relevant header file. + For example, advanced API for version `v0.4` is in `legacy/zstd_v04.h` . -#### Streaming API +#### Obsolete streaming API -Streaming is currently provided by `common/zbuff.h`. +Streaming is now provided within `zstd.h`. +Older streaming API is still provided within `common/zbuff.h`. +It is considered obsolete, and will be removed in a future version. +Consider migrating towards newer streaming API. #### Miscellaneous @@ -55,3 +58,4 @@ The other files are not source code. There are : - LICENSE : contains the BSD license text - Makefile : script to compile or install zstd library (static and dynamic) - libzstd.pc.in : for pkg-config (`make install`) + - README.md : this file diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e218b6374..00b1408ca 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -32,7 +32,7 @@ */ -/* ******************************************************* +/*-******************************************************* * Compiler specifics *********************************************************/ #ifdef _MSC_VER /* Visual Studio */ @@ -55,7 +55,7 @@ #include "mem.h" #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH_reset, update, digest */ -#define FSE_STATIC_LINKING_ONLY +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ #include "fse.h" #define HUF_STATIC_LINKING_ONLY #include "huf.h" @@ -2758,6 +2758,275 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, +/* ****************************************************************** +* Streaming +********************************************************************/ + +typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; + +struct ZSTD_CStream_s { + ZSTD_CCtx* zc; + char* inBuff; + size_t inBuffSize; + size_t inToCompress; + size_t inBuffPos; + size_t inBuffTarget; + size_t blockSize; + char* outBuff; + size_t outBuffSize; + size_t outBuffContentSize; + size_t outBuffFlushedSize; + ZSTD_cStreamStage stage; + U32 checksum; + U32 frameEnded; + ZSTD_customMem customMem; +}; /* typedef'd to ZSTD_CStream within "zstd.h" */ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + return ZSTD_createCStream_advanced(defaultCustomMem); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ + ZSTD_CStream* zcs; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zcs = (ZSTD_CStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CStream)); + if (zcs==NULL) return NULL; + memset(zcs, 0, sizeof(ZSTD_CStream)); + memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); + zcs->zc = ZSTD_createCCtx_advanced(customMem); + if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; } + return zcs; +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + if (zcs==NULL) return 0; /* support free on NULL */ + ZSTD_freeCCtx(zcs->zc); + if (zcs->inBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff); + if (zcs->outBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff); + zcs->customMem.customFree(zcs->customMem.opaque, zcs); + return 0; +} + + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } +size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } + +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + /* allocate buffers */ + { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; + if (zcs->inBuffSize < neededInBuffSize) { + zcs->inBuffSize = neededInBuffSize; + zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff); /* should not be necessary */ + zcs->inBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, neededInBuffSize); + if (zcs->inBuff == NULL) return ERROR(memory_allocation); + } + zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); + } + if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { + zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; + zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff); /* should not be necessary */ + zcs->outBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, zcs->outBuffSize); + if (zcs->outBuff == NULL) return ERROR(memory_allocation); + } + + { size_t const errorCode = ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize); + if (ZSTD_isError(errorCode)) return errorCode; } + + zcs->inToCompress = 0; + zcs->inBuffPos = 0; + zcs->inBuffTarget = zcs->blockSize; + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + zcs->checksum = params.fParams.checksumFlag > 0; + zcs->frameEnded = 0; + return 0; /* ready to go */ +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0); +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel); +} + + +/*====== Compression ======*/ + +typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr, + ZSTD_flush_e const flush) +{ + U32 someMoreWork = 1; + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + + while (someMoreWork) { + switch(zcs->stage) + { + case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ + + case zcss_load: + /* complete inBuffer */ + { size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); + zcs->inBuffPos += loaded; + ip += loaded; + if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) { + someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ + } } + /* compress current block (note : this stage cannot be stopped in the middle) */ + { void* cDst; + size_t cSize; + size_t const iSize = zcs->inBuffPos - zcs->inToCompress; + size_t oSize = oend-op; + if (oSize >= ZSTD_compressBound(iSize)) + cDst = op; /* compress directly into output buffer (avoid flush stage) */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + cSize = (flush == zsf_end) ? + ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); + if (ZSTD_isError(cSize)) return cSize; + if (flush == zsf_end) zcs->frameEnded = 1; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ + zcs->inToCompress = zcs->inBuffPos; + if (cDst == op) { op += cSize; break; } /* no need to flush */ + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_flush; /* pass-through to flush stage */ + } + + case zcss_flush: + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + break; + } + + case zcss_final: + someMoreWork = 0; /* do nothing */ + break; + + default: + return ERROR(GENERIC); /* impossible */ + } + } + + *srcSizePtr = ip - istart; + *dstCapacityPtr = op - ostart; + if (zcs->frameEnded) return 0; + { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; + if (hintInSize==0) hintInSize = zcs->blockSize; + return hintInSize; + } +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + size_t sizeRead = input->size - input->pos; + size_t sizeWritten = output->size - output->pos; + size_t const result = ZSTD_compressStream_generic(zcs, + (char*)(output->dst) + output->pos, &sizeWritten, + (const char*)(input->src) + input->pos, &sizeRead, zsf_gather); + input->pos += sizeRead; + output->pos += sizeWritten; + return result; +} + + +/*====== Finalize ======*/ + +/*! ZSTD_flushStream() : +* @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + size_t srcSize = 0; + size_t sizeWritten = output->size - output->pos; + size_t const result = ZSTD_compressStream_generic(zcs, + (char*)(output->dst) + output->pos, &sizeWritten, + &srcSize, &srcSize, /* use a valid src address instead of NULL */ + zsf_flush); + output->pos += sizeWritten; + if (ZSTD_isError(result)) return result; + return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + BYTE* const ostart = (BYTE*)(output->dst) + output->pos; + BYTE* const oend = (BYTE*)(output->dst) + output->size; + BYTE* op = ostart; + + if (zcs->stage != zcss_final) { + /* flush whatever remains */ + size_t srcSize = 0; + size_t sizeWritten = output->size - output->pos; + size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ + size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + op += sizeWritten; + if (remainingToFlush) { + output->pos += sizeWritten; + return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); + } + /* create epilogue */ + zcs->stage = zcss_final; + zcs->outBuffContentSize = !notEnded ? 0 : + ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ + } + + /* flush epilogue */ + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + op += flushed; + zcs->outBuffFlushedSize += flushed; + output->pos += op-ostart; + if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */ + return toFlush - flushed; + } +} + + + /*-===== Pre-defined compression levels =====-*/ #define ZSTD_DEFAULT_CLEVEL 1 diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 052ccd601..74e166add 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1301,3 +1301,239 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, dst, dstCapacity, src, srcSize); } + + +/*===================================== +* Streaming decompression +*====================================*/ + +typedef enum { zdss_init, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +/* *** Resource management *** */ +struct ZSTD_DStream_s { + ZSTD_DCtx* zd; + ZSTD_frameParams fParams; + ZSTD_dStreamStage stage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t blockSize; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t lhSize; + ZSTD_customMem customMem; +}; /* typedef'd to ZSTD_DStream within "zstd.h" */ + + +ZSTD_DStream* ZSTD_createDStream(void) +{ + return ZSTD_createDStream_advanced(defaultCustomMem); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + ZSTD_DStream* zds; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zds = (ZSTD_DStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DStream)); + if (zds==NULL) return NULL; + memset(zds, 0, sizeof(ZSTD_DStream)); + memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); + zds->zd = ZSTD_createDCtx_advanced(customMem); + if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } + zds->stage = zdss_init; + return zds; +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + if (zds==NULL) return 0; /* support free on null */ + ZSTD_freeDCtx(zds->zd); + if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); + if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); + zds->customMem.customFree(zds->customMem.opaque, zds); + return 0; +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } + +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize); +} + +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + return ZSTD_initDStream_usingDict(zds, NULL, 0); +} + + +/* *** Decompression *** */ + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const istart = (const char*)(input->src) + input->pos; + const char* const iend = (const char*)(input->src) + input->size; + const char* ip = istart; + char* const ostart = (char*)(output->dst) + output->pos; + char* const oend = (char*)(output->dst) + output->size; + char* op = ostart; + U32 someMoreWork = 1; + + while (someMoreWork) { + switch(zds->stage) + { + case zdss_init : + return ERROR(init_missing); + + case zdss_loadHeader : + { size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize); + if (ZSTD_isError(hSize)) return hSize; + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); + zds->lhSize += iend-ip; + input->pos = input->size; + return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */ + size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size); + if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ + if (h1Size < zds->lhSize) { /* long header */ + size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd); + size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size); + if (ZSTD_isError(h2Result)) return h2Result; + } } + + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + + /* Frame header instruct buffer sizes */ + { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zds->fParams.windowSize + blockSize; + zds->blockSize = blockSize; + if (zds->inBuffSize < blockSize) { + zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); + zds->inBuffSize = blockSize; + zds->inBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, blockSize); + if (zds->inBuff == NULL) return ERROR(memory_allocation); + } + if (zds->outBuffSize < neededOutSize) { + zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); + zds->outBuffSize = neededOutSize; + zds->outBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, neededOutSize); + if (zds->outBuff == NULL) return ERROR(memory_allocation); + } } + zds->stage = zdss_read; + /* pass-through */ + + case zdss_read: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + if (neededInSize==0) { /* end of frame */ + zds->stage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); + size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + break; + } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->stage = zdss_load; + /* pass-through */ + } + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); + size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, + zds->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zds->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + /* pass-through */ + } } + + case zdss_flush: + { size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->stage = zdss_read; + if (zds->outStart + zds->blockSize > zds->outBuffSize) + zds->outStart = zds->outEnd = 0; + break; + } + /* cannot flush everything */ + someMoreWork = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + input->pos += (size_t)(ip-istart); + output->pos += (size_t)(op-ostart); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); + if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); + if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ + nextSrcSizeHint -= zds->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} + + + + + + diff --git a/lib/zstd.h b/lib/zstd.h index 7bd8a05c4..056940ff1 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -36,7 +36,7 @@ extern "C" { #endif -/*====== Dependency ======*/ +/*====== Dependency ======*/ #include /* size_t */ @@ -52,7 +52,7 @@ extern "C" { #endif -/*====== Version ======*/ +/*======= Version =======*/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 8 #define ZSTD_VERSION_RELEASE 1 @@ -84,15 +84,13 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, * potentially larger than what local system can handle as a single memory segment. * In which case, it's necessary to use streaming mode to decompress data. * note 2 : decompressed size is an optional field, that may not be present. -* When `return==0`, consider data to decompress could have any size. -* In which case, it's necessary to use streaming mode to decompress data, -* or rely on application's implied limits. -* (For example, it may know that its own data is necessarily cut into blocks <= 16 KB). +* When `return==0`, data to decompress can have any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application may rely on its own implied limits. +* (For example, application own data could be necessarily cut into blocks <= 16 KB). * note 3 : decompressed size could be wrong or intentionally modified ! * Always ensure result fits within application's authorized limits ! -* Each application can have its own set of conditions. -* If the intention is to decompress public data compressed by zstd command line utility, -* it is recommended to support at least 8 MB for extended compatibility. +* Each application can set its own limits. * note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); @@ -100,7 +98,7 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. `dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ). If originalSize is unknown, and if there is no implied application-specific limitations, - it's necessary to use streaming mode to decompress data. + it's preferable to use streaming mode to decompress data. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), or an errorCode if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, @@ -141,7 +139,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa ***************************/ /*! ZSTD_compress_usingDict() : * Compression using a predefined Dictionary (see dictBuilder/zdict.h). -* Note : This function load the dictionary, resulting in a significant startup time. */ +* Note : This function load the dictionary, resulting in significant startup delay. */ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -151,7 +149,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, /*! ZSTD_decompress_usingDict() : * Decompression using a predefined Dictionary (see dictBuilder/zdict.h). * Dictionary must be identical to the one used during compression. -* Note : This function load the dictionary, resulting in a significant startup time */ +* Note : This function load the dictionary, resulting in significant startup delay */ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -163,7 +161,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, ****************************/ /*! ZSTD_createCDict() : * Create a digested dictionary, ready to start compression operation without startup delay. -* `dict` can be released after creation */ +* `dict` can be released after ZSTD_CDict creation */ typedef struct ZSTD_CDict_s ZSTD_CDict; ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); @@ -232,19 +230,19 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */ typedef struct { - unsigned windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ - unsigned chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - unsigned hashLog; /*< dispatch table : larger == faster, more memory */ - unsigned searchLog; /*< nb of searches : larger == more compression, slower */ - unsigned searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */ - unsigned targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */ + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ ZSTD_strategy strategy; } ZSTD_compressionParameters; typedef struct { - unsigned contentSizeFlag; /*< 1: content size will be in frame header (if known). */ - unsigned checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ - unsigned noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */ + unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */ + unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */ } ZSTD_frameParameters; typedef struct { @@ -323,12 +321,122 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx); /* ****************************************************************** -* Buffer-less streaming functions (synchronous mode) +* Streaming +********************************************************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + +/*====== compression ======*/ + +/*-*********************************************************************** +* Streaming compression - howto +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* +* Start by initializing ZSTD_CStream. +* Use ZSTD_initCStream() to start a new compression operation. +* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. +* +* Use ZSTD_compressStream() repetitively to consume input stream. +* The function will automatically update both `pos`. +* Note that it may not consume the entire input, in which case `pos < size`, +* and it's up to the caller to present again remaining data. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream(). +* `output->pos` will be updated. +* Note some content might still be left within internal buffer if `output->size` is too small. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* ZSTD_endStream() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. +* In which case, call again ZSTD_endStream() to complete the flush. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef struct ZSTD_CStream_s ZSTD_CStream; +ZSTD_CStream* ZSTD_createCStream(void); +size_t ZSTD_freeCStream(ZSTD_CStream* zcs); + +size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + +/* advanced */ +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize); + + +/*====== decompression ======*/ + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-init multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation, +* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos`. +* Note that it may not consume the entire input (pos < size), +* in which case it's up to the caller to present remaining input again. +* @return : 0 when a frame is completely decoded and fully flushed, +* 1 when there is still some data left within internal buffer to flush, +* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************************/ + +typedef struct ZSTD_DStream_s ZSTD_DStream; +ZSTD_DStream* ZSTD_createDStream(void); +size_t ZSTD_freeDStream(ZSTD_DStream* zds); + +size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ + +size_t ZSTD_initDStream(ZSTD_DStream* zds); +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +/* advanced */ +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + + + +/* ****************************************************************** +* Buffer-less and synchronous inner streaming functions ********************************************************************/ /* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with a lot of restrictions (documented below). -* For an easier streaming API, look into common/zbuff.h -* which removes all restrictions by allocating and managing its own internal buffer */ +* But it's also a complex one, with many restrictions (documented below). +* Prefer using normal streaming API for an easier experience */ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); diff --git a/programs/.gitignore b/programs/.gitignore index adf78081d..369a27cc1 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -9,6 +9,8 @@ fuzzer fuzzer32 zbufftest zbufftest32 +zstreamtest +zstreamtest32 datagen paramgrill paramgrill32 diff --git a/programs/Makefile b/programs/Makefile index 9a9fcd346..a51d438b5 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -94,7 +94,7 @@ all32: cleano32 all $(ZSTDDIR)/decompress/zstd_decompress.o: CFLAGS += $(ALIGN_LOOP) -zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \ +zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \ zstdcli.c fileio.c bench.c datagen.c dibio.c $(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT) @@ -116,15 +116,15 @@ zstd-pgo : clean zstd rm zstd $(MAKE) zstd MOREFLAGS=-fprofile-use -zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c +zstd-frugal: $(ZSTD_FILES) zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT) zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \ - $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c + zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \ - $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c + zstdcli.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT) zstd-small: clean @@ -152,6 +152,13 @@ zbufftest32 : CFLAGS += -m32 zbufftest32 : EXT := 32$(EXT) zbufftest32 : zbufftest +zstreamtest : $(ZSTD_FILES) datagen.c zstreamtest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +zstreamtest32 : CFLAGS += -m32 +zstreamtest32 : EXT := 32$(EXT) +zstreamtest32 : zstreamtest + paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) @@ -229,9 +236,9 @@ ifneq (,$(filter $(HOST_OS),MSYS POSIX)) zstd-playTests: datagen ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zbuff +test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream -test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 +test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32 test-all: test test32 valgrindTest @@ -263,4 +270,11 @@ test-zbuff: zbufftest test-zbuff32: zbufftest32 ./zbufftest32 $(ZBUFFTEST) + +test-zstream: zstreamtest + ./zstreamtest $(ZBUFFTEST) + +test-zstream32: zstreamtest32 + ./zstreamtest32 $(ZBUFFTEST) + endif diff --git a/programs/fileio.c b/programs/fileio.c index b04ee3b04..04d5a71ac 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -22,8 +22,7 @@ - zstd homepage : http://www.zstd.net */ /* - Note : this is stand-alone program. - It is not part of ZSTD compression library, it is a user program of ZSTD library. + Note : this file is part of zstd command line, which is not library. The license of ZSTD library is BSD. The license of this file is GPLv2. */ @@ -41,8 +40,11 @@ /* ************************************* * Compiler Options ***************************************/ -#define _POSIX_SOURCE 1 /* enable %llu on Windows */ -#define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ +#ifdef _MSC_VER /* Visual */ +# define _POSIX_SOURCE 1 /* enable %llu on Windows */ +# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ +# pragma warning(disable : 4204) /* non-constant aggregate initializer */ +#endif /*-************************************* @@ -59,8 +61,6 @@ #include "fileio.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" -#define ZBUFF_STATIC_LINKING_ONLY -#include "zbuff.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) # include "zstd_legacy.h" /* ZSTD_isLegacy */ @@ -173,10 +173,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName) SET_BINARY_MODE(stdin); } else { f = fopen(srcFileName, "rb"); + if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); } - if ( f==NULL ) DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); - return f; } @@ -212,9 +211,9 @@ static FILE* FIO_openDstFile(const char* dstFileName) while ((ch!=EOF) && (ch!='\n')) ch = getchar(); /* flush rest of input line */ } } } f = fopen( dstFileName, "wb" ); + if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); } - if (f==NULL) DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); return f; } @@ -265,7 +264,7 @@ typedef struct { size_t dstBufferSize; void* dictBuffer; size_t dictBufferSize; - ZBUFF_CCtx* ctx; + ZSTD_CStream* cctx; FILE* dstFile; FILE* srcFile; } cRess_t; @@ -275,11 +274,11 @@ static cRess_t FIO_createCResources(const char* dictFileName) cRess_t ress; memset(&ress, 0, sizeof(ress)); - ress.ctx = ZBUFF_createCCtx(); - if (ress.ctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZBUFF context"); - ress.srcBufferSize = ZBUFF_recommendedCInSize(); + ress.cctx = ZSTD_createCStream(); + if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream"); + ress.srcBufferSize = ZSTD_CStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZBUFF_recommendedCOutSize(); + ress.dstBufferSize = ZSTD_CStreamOutSize(); ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory"); @@ -295,8 +294,8 @@ static void FIO_freeCResources(cRess_t ress) free(ress.srcBuffer); free(ress.dstBuffer); free(ress.dictBuffer); - errorCode = ZBUFF_freeCCtx(ress.ctx); - if (ZBUFF_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode)); + errorCode = ZSTD_freeCStream(ress.cctx); + if (ZSTD_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZSTD_CStream : %s", ZSTD_getErrorName(errorCode)); } @@ -324,8 +323,8 @@ static int FIO_compressFilename_internal(cRess_t ress, params.cParams.windowLog = g_maxWLog; params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize); } - { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); - if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); + { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); + if (ZSTD_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(errorCode)); } } /* Main compression loop */ @@ -337,30 +336,30 @@ static int FIO_compressFilename_internal(cRess_t ress, DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20)); /* Compress using buffered streaming */ - { size_t usedInSize = inSize; - size_t cSize = ress.dstBufferSize; - { size_t const result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize); - if (ZBUFF_isError(result)) EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result)); } - if (inSize != usedInSize) + { ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; + ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 }; + { size_t const result = ZSTD_compressStream(ress.cctx, &outBuff, &inBuff); + if (ZSTD_isError(result)) EXM_THROW(23, "Compression error : %s ", ZSTD_getErrorName(result)); } + if (inBuff.pos != inBuff.size) /* inBuff should be entirely consumed since buffer sizes are recommended ones */ EXM_THROW(24, "Compression error : input block not fully consumed"); /* Write cBlock */ - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); } - compressedfilesize += cSize; + { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); + if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); } + compressedfilesize += outBuff.pos; } DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100); } /* End of Frame */ - { size_t cSize = ress.dstBufferSize; - size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize); + { ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; + size_t const result = ZSTD_endStream(ress.cctx, &outBuff); if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end"); - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile); - if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); } - compressedfilesize += cSize; + { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); + if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); } + compressedfilesize += outBuff.pos; } /* Status */ @@ -496,7 +495,7 @@ typedef struct { size_t dstBufferSize; void* dictBuffer; size_t dictBufferSize; - ZBUFF_DCtx* dctx; + ZSTD_DStream* dctx; FILE* dstFile; } dRess_t; @@ -506,11 +505,11 @@ static dRess_t FIO_createDResources(const char* dictFileName) memset(&ress, 0, sizeof(ress)); /* Allocation */ - ress.dctx = ZBUFF_createDCtx(); - if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context"); - ress.srcBufferSize = ZBUFF_recommendedDInSize(); + ress.dctx = ZSTD_createDStream(); + if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream"); + ress.srcBufferSize = ZSTD_DStreamInSize(); ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZBUFF_recommendedDOutSize(); + ress.dstBufferSize = ZSTD_DStreamOutSize(); ress.dstBuffer = malloc(ress.dstBufferSize); if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); @@ -522,8 +521,8 @@ static dRess_t FIO_createDResources(const char* dictFileName) static void FIO_freeDResources(dRess_t ress) { - size_t const errorCode = ZBUFF_freeDCtx(ress.dctx); - if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode)); + size_t const errorCode = ZSTD_freeDStream(ress.dctx); + if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", ZSTD_getErrorName(errorCode)); free(ress.srcBuffer); free(ress.dstBuffer); free(ress.dictBuffer); @@ -615,7 +614,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, size_t readSize; U32 storedSkips = 0; - ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); + ZSTD_initDStream_usingDict(ress.dctx, ress.dictBuffer, ress.dictBufferSize); /* Header loading (optional, saves one loop) */ { size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */ @@ -625,18 +624,18 @@ unsigned long long FIO_decompressFrame(dRess_t ress, /* Main decompression Loop */ while (1) { - size_t inSize=readSize, decodedSize=ress.dstBufferSize; - size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); - if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); - readSize -= inSize; + ZSTD_inBuffer inBuff = { ress.srcBuffer, readSize, 0 }; + ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 }; + size_t const toRead = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff ); + if (ZSTD_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(toRead)); /* Write block */ - storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, decodedSize, storedSkips); - frameSize += decodedSize; + storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, outBuff.pos, storedSkips); + frameSize += outBuff.pos; DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); if (toRead == 0) break; /* end of frame */ - if (readSize) EXM_THROW(37, "Decoding error : should consume entire input"); + if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input"); /* Fill input buffer */ if (toRead > ress.srcBufferSize) EXM_THROW(38, "too large block"); diff --git a/programs/zstreamtest.c b/programs/zstreamtest.c new file mode 100644 index 000000000..7f136b1b5 --- /dev/null +++ b/programs/zstreamtest.c @@ -0,0 +1,659 @@ +/* + Fuzzer test tool for zstd streaming API + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - ZSTD homepage : https://www.zstd.net/ +*/ + +/*-************************************ +* Compiler specific +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +#endif + + +/*-************************************ +* Includes +**************************************/ +#include /* free */ +#include /* fgets, sscanf */ +#include /* timeb */ +#include /* strcmp */ +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ +#include "zstd.h" /* ZSTD_compressBound */ +#include "datagen.h" /* RDG_genBuffer */ +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" /* XXH64_* */ + + +/*-************************************ +* Constants +**************************************/ +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +static const U32 nbTestsDefault = 10000; +#define COMPRESSIBLE_NOISE_LENGTH (10 MB) +#define FUZ_COMPRESSIBILITY_DEFAULT 50 +static const U32 prime1 = 2654435761U; +static const U32 prime2 = 2246822519U; + + + +/*-************************************ +* Display Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \ + { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stdout); } } +static const U32 g_refreshRate = 150; +static U32 g_displayTime = 0; + +static U32 g_testTime = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +#define MAX(a,b) ((a)>(b)?(a):(b)) + +static U32 FUZ_GetMilliStart(void) +{ + struct timeb tb; + U32 nCount; + ftime( &tb ); + nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm); + return nCount; +} + + +static U32 FUZ_GetMilliSpan(U32 nTimeStart) +{ + U32 const nCurrent = FUZ_GetMilliStart(); + U32 nSpan = nCurrent - nTimeStart; + if (nTimeStart > nCurrent) + nSpan += 0x100000 * 1000; + return nSpan; +} + +/*! FUZ_rand() : + @return : a 27 bits random value, from a 32-bits `seed`. + `seed` is also modified */ +# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +unsigned int FUZ_rand(unsigned int* seedPtr) +{ + U32 rand32 = *seedPtr; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *seedPtr = rand32; + return rand32 >> 5; +} + + +/* +static unsigned FUZ_highbit32(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + for ( ; v32 ; v32>>=1) nbBits++; + return nbBits; +} +*/ + +static void* allocFunction(void* opaque, size_t size) +{ + void* address = malloc(size); + (void)opaque; + return address; +} + +static void freeFunction(void* opaque, void* address) +{ + (void)opaque; + free(address); +} + +static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) +{ + int testResult = 0; + size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; + void* CNBuffer = malloc(CNBufferSize); + size_t const skippableFrameSize = 11; + size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + void* compressedBuffer = malloc(compressedBufferSize); + size_t const decodedBufferSize = CNBufferSize; + void* decodedBuffer = malloc(decodedBufferSize); + size_t cSize; + U32 testNb=0; + ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); + ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); + ZSTD_inBuffer inBuff; + ZSTD_outBuffer outBuff; + + /* Create compressible test buffer */ + if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { + DISPLAY("Not enough memory, aborting\n"); + goto _output_error; + } + RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + + /* generate skippable frame */ + MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); + cSize = skippableFrameSize + 8; + + /* Basic compression test */ + DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1); + outBuff.dst = (char*)(compressedBuffer)+cSize; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); + if (ZSTD_isError(r)) goto _output_error; } + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (r != 0) goto _output_error; } /*< error, or some data not flushed */ + cSize += outBuff.pos; + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + /* skippable frame test */ + DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); + ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (r != 0) goto _output_error; } + if (outBuff.pos != 0) goto _output_error; /* skippable frame len is 0 */ + DISPLAYLEVEL(4, "OK \n"); + + /* Basic decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + /* check regenerated data is byte exact */ + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + { size_t i; + for (i=0; i "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + +static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) +{ + static const U32 maxSrcLog = 24; + static const U32 maxSampleLog = 19; + BYTE* cNoiseBuffer[5]; + size_t srcBufferSize = (size_t)1<= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); + FUZ_rand(&coreSeed); + lseed = coreSeed ^ prime1; + + /* states full reset (unsynchronized) */ + /* some issues only happen when reusing states in a specific sequence of parameters */ + if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); } + if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); } + + /* srcBuffer selection [0-4] */ + { U32 buffNb = FUZ_rand(&lseed) & 0x7F; + if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ + else { + buffNb >>= 3; + if (buffNb & 7) { + const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ + buffNb = tnb[buffNb >> 3]; + } else { + const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ + buffNb = tnb[buffNb >> 3]; + } } + srcBuffer = cNoiseBuffer[buffNb]; + } + + /* compression init */ + { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; + maxTestSize = FUZ_rLogLength(&lseed, testLog); + dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; + /* random dictionary selection */ + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; + } + { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize); + params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; + params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; + { size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0); + CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError)); + } } } + + /* multi-segments compression test */ + XXH64_reset(&xxhState, 0); + nbChunks = (FUZ_rand(&lseed) & 127) + 2; + { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; + for (n=0, cSize=0, totalTestSize=0 ; (n= remainingToFlush); + outBuff.size = outBuff.pos + adjustedDstSize; + remainingToFlush = ZSTD_endStream(zc, &outBuff); + CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush)); + CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush); + } } + crcOrig = XXH64_digest(&xxhState); + cSize = outBuff.pos; + } + + /* multi - fragments decompression test */ + ZSTD_initDStream_usingDict(zd, dict, dictSize); + { size_t decompressionResult = 1; + ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + for (totalGenSize = 0 ; decompressionResult ; ) { + size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); + inBuff.size = inBuff.pos + readCSrcSize; + outBuff.size = inBuff.pos + dstBuffSize; + decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); + CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); + } + CHECK (decompressionResult != 0, "frame not fully decoded"); + CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size") + CHECK (inBuff.pos != cSize, "compressed data should be fully read") + { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); + if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); + CHECK (crcDest!=crcOrig, "decompressed data corrupted"); + } } + + /*===== noisy/erroneous src decompression test =====*/ + + /* add some noise */ + { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; + U32 nn; for (nn=0; nn='0') && (*argument<='9')) { + nbTests *= 10; + nbTests += *argument - '0'; + argument++; + } + break; + + case 'T': + argument++; + nbTests=0; g_testTime=0; + while ((*argument>='0') && (*argument<='9')) { + g_testTime *= 10; + g_testTime += *argument - '0'; + argument++; + } + if (*argument=='m') g_testTime *=60, argument++; + if (*argument=='n') argument++; + g_testTime *= 1000; + break; + + case 's': + argument++; + seed=0; + seedset=1; + while ((*argument>='0') && (*argument<='9')) { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + + case 't': + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; + + case 'P': /* compressibility % */ + argument++; + proba=0; + while ((*argument>='0') && (*argument<='9')) { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba<0) proba=0; + if (proba>100) proba=100; + break; + + default: + return FUZ_usage(programName); + } + } } } /* for(argNb=1; argNbavail_out; if (zwc->bytesLeft) { @@ -246,6 +246,18 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (flush == Z_FINISH && bytesLeft == 0) return Z_STREAM_END; zwc->bytesLeft = bytesLeft; } + + if (flush == Z_SYNC_FLUSH) { + size_t bytesLeft; + size_t dstCapacity = strm->avail_out; + bytesLeft = ZBUFF_compressFlush(zwc->zbc, strm->next_out, &dstCapacity); + LOG_WRAPPER("ZBUFF_compressFlush avail_out=%d dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)dstCapacity, (int)bytesLeft); + if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR; + strm->next_out += dstCapacity; + strm->total_out += dstCapacity; + strm->avail_out -= dstCapacity; + zwc->bytesLeft = bytesLeft; + } return Z_OK; }