From 82613d72e4df2449ebc1e67f81e29ef8a52c8a8a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 26 Oct 2016 18:10:43 -0700 Subject: [PATCH] added example multiple_streaming_compression --- examples/.gitignore | 1 + examples/Makefile | 12 +- examples/README.md | 5 + examples/multiple_streaming_compression.c | 163 ++++++++++++++++++++++ 4 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 examples/multiple_streaming_compression.c diff --git a/examples/.gitignore b/examples/.gitignore index 1c98e1884..0711813d3 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -5,6 +5,7 @@ dictionary_compression dictionary_decompression streaming_compression streaming_decompression +multiple_streaming_compression #test artefact tmp* diff --git a/examples/Makefile b/examples/Makefile index 8ce6a2589..741022869 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -17,7 +17,8 @@ default: all all: simple_compression simple_decompression \ dictionary_compression dictionary_decompression \ - streaming_compression streaming_decompression + streaming_compression streaming_decompression \ + multiple_streaming_compression simple_compression : simple_compression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ @@ -34,6 +35,9 @@ dictionary_decompression : dictionary_decompression.c streaming_compression : streaming_compression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +multiple_streaming_compression : multiple_streaming_compression.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + streaming_decompression : streaming_decompression.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ @@ -41,7 +45,8 @@ clean: @rm -f core *.o tmp* result* *.zst \ simple_compression simple_decompression \ dictionary_compression dictionary_decompression \ - streaming_compression streaming_decompression + streaming_compression streaming_decompression \ + multiple_streaming_compression @echo Cleaning completed test: all @@ -54,7 +59,10 @@ test: all @echo starting streaming compression ./streaming_compression tmp ./streaming_decompression tmp.zst > /dev/null + @echo starting multiple streaming compression + ./multiple_streaming_compression *.c @echo starting dictionary compression ./dictionary_compression tmp2 tmp README.md ./dictionary_decompression tmp2.zst tmp.zst README.md + $(RM) tmp* *.zst @echo tests completed diff --git a/examples/README.md b/examples/README.md index ba132f6c3..8a40443ea 100644 --- a/examples/README.md +++ b/examples/README.md @@ -15,6 +15,11 @@ Zstandard library : usage examples Compress a single file. Introduces usage of : `ZSTD_compressStream()` +- [Multiple Streaming compression](multiple_streaming_compression.c) : + Compress multiple files in a single command line. + Introduces memory usage preservation technique, + reducing impact of malloc()/free() and memset() by re-using existing resources. + - [Streaming decompression](streaming_decompression.c) : Decompress a single file compressed by zstd. Compatible with both simple and streaming compression. diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c new file mode 100644 index 000000000..81541f590 --- /dev/null +++ b/examples/multiple_streaming_compression.c @@ -0,0 +1,163 @@ +/** + * Copyright 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE-examples file in the root directory of this source tree. + */ + + +/* The objective of this example is to show of to compress multiple successive files +* while preserving memory management. +* All structures and buffers will be created only once, +* and shared across all compression operations */ + +#include // malloc, exit +#include // fprintf, perror, feof +#include // strerror +#include // errno +#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being +#include // presumes zstd library is installed + + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc:"); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(5); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + + +typedef struct { + void* buffIn; + void* buffOut; + size_t buffInSize; + size_t buffOutSize; + ZSTD_CStream* cstream; +} resources ; + +static resources createResources_orDie() +{ + resources ress; + ress.buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */ + ress.buffOutSize= ZSTD_CStreamOutSize(); /* can always flush a full block */ + ress.buffIn = malloc_orDie(ress.buffInSize); + ress.buffOut= malloc_orDie(ress.buffOutSize); + ress.cstream = ZSTD_createCStream(); + if (ress.cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } + return ress; +} + +static void freeResources(resources ress) +{ + ZSTD_freeCStream(ress.cstream); + free(ress.buffIn); + free(ress.buffOut); +} + + +static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + FILE* const fout = fopen_orDie(outName, "wb"); + + size_t const initResult = ZSTD_initCStream(ress.cstream, cLevel); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } + + size_t read, toRead = ress.buffInSize; + while( (read = fread_orDie(ress.buffIn, toRead, fin)) ) { + ZSTD_inBuffer input = { ress.buffIn, read, 0 }; + while (input.pos < input.size) { + ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 }; + toRead = ZSTD_compressStream(ress.cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ + if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } + if (toRead > ress.buffInSize) toRead = ress.buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */ + fwrite_orDie(ress.buffOut, output.pos, fout); + } + } + + ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 }; + size_t const remainingToFlush = ZSTD_endStream(ress.cstream, &output); /* close frame */ + if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); } + fwrite_orDie(ress.buffOut, output.pos, fout); + + fclose_orDie(fout); + fclose_orDie(fin); + + /* success */ + printf("%-30s -> %-30s \n", fname, outName); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc<2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE(s)\n", exeName); + return 1; + } + + resources ress = createResources_orDie(); + void* ofnBuffer = NULL; + size_t ofnbSize = 0; + + int argNb; + for (argNb = 1; argNb < argc; argNb++) { + const char* const ifn = argv[argNb]; + size_t const ifnSize = strlen(ifn); + size_t const ofnSize = ifnSize + 5; + if (ofnbSize <= ofnSize) { + ofnbSize = ofnSize + 16; + free(ofnBuffer); + ofnBuffer = malloc_orDie(ofnbSize); + } + memset(ofnBuffer, 0, ofnSize); + strcat(ofnBuffer, ifn); + strcat(ofnBuffer, ".zst"); + compressFile_orDie(ress, ifn, ofnBuffer, 7); + } + + freeResources(ress); + return 0; +}