From 553b213adae43510c77325f13876c4f5925f2d28 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 12 Aug 2016 18:42:25 +0200 Subject: [PATCH] added streaming_compression.c example --- examples/.gitignore | 1 + examples/Makefile | 16 ++-- examples/streaming_compression.c | 143 +++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 examples/streaming_compression.c diff --git a/examples/.gitignore b/examples/.gitignore index 9d241dba6..3b7352c92 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -3,6 +3,7 @@ simple_compression simple_decompression dictionary_compression dictionary_decompression +streaming_compression #test artefact tmp* diff --git a/examples/Makefile b/examples/Makefile index 6ba8655b4..de1f0d7bb 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -31,7 +31,8 @@ LDFLAGS+= -lzstd default: all all: simple_compression simple_decompression \ - dictionary_compression dictionary_decompression + dictionary_compression dictionary_decompression \ + streaming_compression simple_compression : simple_compression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ @@ -45,19 +46,24 @@ dictionary_compression : dictionary_compression.c dictionary_decompression : dictionary_decompression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +streaming_compression : streaming_compression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + clean: @rm -f core *.o tmp* result* *.zst \ simple_compression simple_decompression \ - dictionary_compression dictionary_decompression + dictionary_compression dictionary_decompression \ + streaming_compression @echo Cleaning completed test: all cp README.md tmp + @echo starting simple compression ./simple_compression tmp - @echo starting simple_decompression ./simple_decompression tmp.zst - @echo dictionary compression + @echo starting streaming compression + ./streaming_compression tmp + @echo starting dictionary compression ./dictionary_compression tmp README.md - @echo dictionary decompression ./dictionary_decompression tmp.zst README.md @echo tests completed diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c new file mode 100644 index 000000000..9082e38a8 --- /dev/null +++ b/examples/streaming_compression.c @@ -0,0 +1,143 @@ +/* + Streaming compression + Educational program using zstd library + Copyright (C) Yann Collet 2016 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + +#include // malloc, exit +#include // fprintf, perror +#include // strerror +#include // errno +#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being +#include // presumes zstd library is installed + + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror(NULL); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(5); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + + +static void compressFile_orDie(const char* fname, const char* outName, int cLevel) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + FILE* const fout = fopen_orDie(outName, "wb"); + size_t const buffInSize = ZSTD_CStreamInSize();; + void* const buffIn = malloc_orDie(buffInSize); + size_t const buffOutSize = ZSTD_CStreamOutSize();; + void* const buffOut = malloc_orDie(buffOutSize); + size_t read, toRead = buffInSize; + + ZSTD_CStream* const cstream = ZSTD_createCStream(); + if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } + size_t const initResult = ZSTD_initCStream(cstream, cLevel); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); } + + while( (read = fread_orDie(buffIn, toRead, fin)) ) { + ZSTD_rCursor cursin = { buffIn, read }; + while (cursin.size) { + ZSTD_wCursor cursout = { buffOut, buffOutSize, 0 }; + toRead = ZSTD_compressStream(cstream, &cursout , &cursin); + fwrite_orDie(buffOut, cursout.nbBytesWritten, fout); + } + } + + ZSTD_wCursor cursout = { buffOut, buffOutSize, 0 }; + size_t const remainingToFlush = ZSTD_endStream(cstream, &cursout); + if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); } + fwrite_orDie(buffOut, cursout.nbBytesWritten, fout); + + fclose_orDie(fout); + fclose_orDie(fin); + free(buffIn); + free(buffOut); +} + + +static const char* createOutFilename_orDie(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* outSpace = malloc_orDie(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (const char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + const char* const inFilename = argv[1]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + const char* const outFilename = createOutFilename_orDie(inFilename); + compressFile_orDie(inFilename, outFilename, 1); + + return 0; +}