refactor utils.h and implement simple_compressionCCtx.c

examples/.gitignore

@@ -1,6 +1,7 @@
 #build
 simple_compression
 simple_decompression
+simple_compressionCCtx
 dictionary_compression
 dictionary_decompression
 streaming_compression

examples/Makefile

@@ -17,6 +17,7 @@ LIB = ../lib/libzstd.a
 default: all
 
 all: simple_compression simple_decompression \
+	simple_compressionCCtx\
 	dictionary_compression dictionary_decompression \
 	streaming_compression streaming_decompression \
 	multiple_streaming_compression streaming_memory_usage
@@ -30,6 +31,9 @@ simple_compression : simple_compression.c $(LIB)
 simple_decompression : simple_decompression.c $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
+simple_compressionCCtx : simple_compressionCCtx.c $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
 dictionary_compression : dictionary_compression.c $(LIB)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
 
@@ -51,6 +55,7 @@ streaming_memory_usage : streaming_memory_usage.c $(LIB)
 clean:
 	@rm -f core *.o tmp* result* *.zst \
         simple_compression simple_decompression \
+	simple_compressionCCtx \
         dictionary_compression dictionary_decompression \
         streaming_compression streaming_decompression \
         multiple_streaming_compression streaming_memory_usage
@@ -62,6 +67,7 @@ test: all
 	@echo -- Simple compression tests
 	./simple_compression tmp
 	./simple_decompression tmp.zst
+	./simple_compressionCCtx *.c
 	./streaming_decompression tmp.zst > /dev/null
 	@echo -- Streaming memory usage
 	./streaming_memory_usage

examples/dictionary_compression.c

@@ -21,7 +21,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
 {
     size_t dictSize;
     printf("loading dictionary %s \n", dictFileName);
-    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize, 0, 0);
     ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
     if (!cdict) {
         fprintf(stderr, "ZSTD_createCDict error \n");
@@ -35,7 +35,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
 static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
 {
     size_t fSize;
-    void* const fBuff = loadFile_orDie(fname, &fSize);
+    void* const fBuff = loadFile_orDie(fname, &fSize, 0, 0);
     size_t const cBuffSize = ZSTD_compressBound(fSize);
     void* const cBuff = malloc_orDie(cBuffSize);
 

examples/dictionary_decompression.c

@@ -25,7 +25,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
 {
     size_t dictSize;
     printf("loading dictionary %s \n", dictFileName);
-    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize, 0, 0);
     ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
     if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); }
     free(dictBuffer);
@@ -35,7 +35,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
 static void decompress(const char* fname, const ZSTD_DDict* ddict)
 {
     size_t cSize;
-    void* const cBuff = loadFile_orDie(fname, &cSize);
+    void* const cBuff = loadFile_orDie(fname, &cSize, 0, 0);
     unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
     if (rSize==ZSTD_CONTENTSIZE_ERROR) {
         fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);

examples/simple_compression.c

@@ -19,7 +19,7 @@
 static void compress_orDie(const char* fname, const char* oname)
 {
     size_t fSize;
-    void* const fBuff = loadFile_orDie(fname, &fSize);
+    void* const fBuff = loadFile_orDie(fname, &fSize, 0, 0);
     size_t const cBuffSize = ZSTD_compressBound(fSize);
     void* const cBuff = malloc_orDie(cBuffSize);
 

examples/simple_compressionCCtx.c

@@ -16,14 +16,14 @@
 #include <zstd.h>      // presumes zstd library is installed
 #include "utils.h"
 
-static void compress_orDie(const char* fname, const char* oname)
+/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/
+static void compressExpress_orDie(const char* fname, const char* oname,
+                                   ZSTD_CCtx* cctx, void* cBuff, size_t cBuffSize, void* fBuff, size_t fBuffSize)
 {
     size_t fSize;
-    void* const fBuff = loadFile_orDie(fname, &fSize);
-    size_t const cBuffSize = ZSTD_compressBound(fSize);
-    void* const cBuff = malloc_orDie(cBuffSize);
+    loadFile_orDie(fname, &fSize, fBuff, fBuffSize);
 
-    size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
+    size_t const cSize = ZSTD_compressCCtx(cctx, cBuff, cBuffSize, fBuff, fBuffSize, 1);
     if (ZSTD_isError(cSize)) {
         fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
         exit(8);
@@ -33,48 +33,65 @@ static void compress_orDie(const char* fname, const char* oname)
 
     /* success */
     printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
-
-    free(fBuff);
-    free(cBuff);
 }
 
-static char* createOutFilename_orDie(const char* filename)
+static void getOutFilename(const char* const filename, char* const outFilename)
 {
-    size_t const inL = strlen(filename);
-    size_t const outL = inL + 5;
-    void* const outSpace = malloc_orDie(outL);
-    memset(outSpace, 0, outL);
-    strcat(outSpace, filename);
-    strcat(outSpace, ".zst");
-    return (char*)outSpace;
+    memset(outFilename, 0, 1);
+    strcat(outFilename, filename);
+    strcat(outFilename, ".zst");
 }
 
 int main(int argc, const char** argv)
 {
     const char* const exeName = argv[0];
 
-    if (argc!=2) {
+    if (argc<2) {
         printf("wrong arguments\n");
         printf("usage:\n");
-        printf("%s FILE\n", exeName);
+        printf("%s FILE(s)\n", exeName);
         return 1;
     }
 
-    const char* const inFilename = argv[1];
+    /* pre-calculate buffer sizes needed to handle all files */
+    size_t maxFileNameLength=0;
+    size_t maxFileSize = 0;
+    size_t maxCBufferSize = 0;
 
-    /** copied code
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+      const char* const fileName = argv[argNb];
+      size_t const fileNameLength = strlen(fileName);
+      size_t const fileSize = fsize_orDie(fileName);
+
+      if (fileNameLength > maxFileNameLength) maxFileNameLength = fileNameLength;
+      if (fileSize > maxFileSize) maxFileSize = fileSize;
+    }
+    maxCBufferSize = ZSTD_compressBound(maxFileSize);
+
+    /* allocate memory for output file name, input/output buffers for all compression tasks */
+    char* const outFilename = (char*)malloc_orDie(maxFileNameLength + 5);
+    void* const fBuffer = malloc_orDie(maxFileSize);
+    void* const cBuffer = malloc_orDie(maxCBufferSize);
+
+    /* create a compression context (ZSTD_CCtx) for all compression tasks */
     ZSTD_CCtx* const cctx = ZSTD_createCCtx();
     if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); }
-    size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
-    if (ZSTD_isError(cSize)) { 
-        fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
-        exit(7);
+
+    /* compress files with shared context, input and output buffers */
+    for (argNb = 1; argNb < argc; argNb++) {
+      const char* const inFilename = argv[argNb];
+      getOutFilename(inFilename, outFilename);
+      compressExpress_orDie(inFilename, outFilename, cctx, cBuffer, maxCBufferSize, fBuffer, maxFileSize);
     }
-    **/
 
-
-    char* const outFilename = createOutFilename_orDie(inFilename);
-    compress_orDie(inFilename, outFilename);
+    /* free momery resources */
     free(outFilename);
+    free(fBuffer);
+    free(cBuffer);
+    ZSTD_freeCCtx(cctx);   /* never fails */
+
+    printf("compressed %i files \n", argc-1);
+
     return 0;
 }

examples/simple_decompression.c

@@ -20,7 +20,7 @@
 static void decompress(const char* fname)
 {
     size_t cSize;
-    void* const cBuff = loadFile_orDie(fname, &cSize);
+    void* const cBuff = loadFile_orDie(fname, &cSize, 0, 0);
     unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
     if (rSize==ZSTD_CONTENTSIZE_ERROR) {
         fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);

examples/utils.h

@@ -40,13 +40,25 @@ typedef enum {
  *
  * @return The size of a given file path.
  */
-static off_t fsize_orDie(const char *filename)
+static size_t fsize_orDie(const char *filename)
 {
     struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
-    /* error */
-    perror(filename);
-    exit(ERROR_fsize);
+    if (stat(filename, &st) != 0) {
+        /* error */
+        perror(filename);
+        exit(ERROR_fsize);
+    }
+
+    off_t const fileSize = st.st_size;
+    size_t const size = (size_t)fileSize;
+    /* if off_t -> size_t conversion causes discrepancy, the file size is
+     * too big for at least 1 type to handle
+     */
+    if (size != fileSize) {   /* narrowcast overflow */
+        fprintf(stderr, "%s : filesize too large \n", filename);
+        exit(ERROR_largeFile);
+    }
+    return size;
 }
 
 /*! fopen_orDie() :
@@ -128,7 +140,8 @@ static void* malloc_orDie(size_t size)
 }
 
 /*! loadFile_orDie() :
- * Read size bytes from a file.
+ * Read size bytes from a file. If buffer is not provided (i.e., buffer == null),
+ * malloc will be called to allocate one.
  *
  * Note: This function will send an error to stderr and exit if it
  * cannot read data from the given file path.
@@ -136,23 +149,24 @@ static void* malloc_orDie(size_t size)
  * @return If successful this function will return a pointer to read
  * data otherwise it will printout an error to stderr and exit.
  */
-static void* loadFile_orDie(const char* fileName, size_t* size)
+static void* loadFile_orDie(const char* fileName, size_t* size, void* buffer, int bufferSize)
 {
-    off_t const fileSize = fsize_orDie(fileName);
-    size_t const buffSize = (size_t)fileSize;
-    if ((off_t)buffSize < fileSize) {   /* narrowcast overflow */
-        fprintf(stderr, "%s : filesize too large \n", fileName);
-        exit(ERROR_largeFile);
-    }
+    size_t const fileSize = fsize_orDie(fileName);
     FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
+    if (!buffer) {
+        buffer = malloc_orDie(fileSize);
+    }
+    else if (bufferSize < fileSize) {
+        fprintf(stderr, "%s : filesize bigger than provided buffer.\n", fileName);
+
+    }
+    size_t const readSize = fread(buffer, 1, fileSize, inFile);
+    if (readSize != (size_t)fileSize) {
         fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
         exit(ERROR_fread);
     }
     fclose(inFile);  /* can't fail, read only */
-    *size = buffSize;
+    *size = fileSize;
     return buffer;
 }