From f16f4497ca121f8a0ec82e2af35a4a00d82acf13 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 9 May 2017 16:18:17 -0700 Subject: [PATCH] added ZSTD_estimateDStreamSize() --- doc/zstd_manual.html | 15 ++++++++------- lib/decompress/zstd_decompress.c | 30 ++++++++++++++++++++---------- lib/zstd.h | 15 ++++++++------- tests/zstreamtest.c | 15 ++++++++++++++- 4 files changed, 50 insertions(+), 25 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index d24a6b892..9cfb1b68b 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -321,6 +321,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB ZSTD_frameParameters fParams; } ZSTD_parameters;
+
typedef struct {
+    unsigned long long frameContentSize;
+    unsigned windowSize;
+    unsigned dictID;
+    unsigned checksumFlag;
+} ZSTD_frameHeader;
+

Custom memory allocation functions

typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
@@ -392,6 +399,7 @@ size_t ZSTD_estimateDCtxSize(void);
 


 
 
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
+size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
          an internal ?Dict will be created, which size is not estimated.
          In this case, get additional size by using ZSTD_estimate?DictSize 
@@ -638,13 +646,6 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned lo
   It also returns Frame Size as fparamsPtr->frameContentSize.
 

 
-
typedef struct {
-    unsigned long long frameContentSize;
-    unsigned windowSize;
-    unsigned dictID;
-    unsigned checksumFlag;
-} ZSTD_frameHeader;
-


 
Buffer-less streaming decompression functions
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 3c231a941..267aa94d0 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -221,20 +221,21 @@ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
 
 /** ZSTD_getFrameHeader() :
 *   decode Frame Header, or require larger `srcSize`.
-*   @return : 0, `fparamsPtr` is correctly filled,
+*   @return : 0, `fhiPtr` is correctly filled,
 *            >0, `srcSize` is too small, result is expected `srcSize`,
 *             or an error code, which can be tested using ZSTD_isError() */
-size_t ZSTD_getFrameHeader(ZSTD_frameHeader* fparamsPtr, const void* src, size_t srcSize)
+size_t ZSTD_getFrameHeader(ZSTD_frameHeader* fhiPtr, const void* src, size_t srcSize)
 {
     const BYTE* ip = (const BYTE*)src;
-
     if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix;
+
     if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
         if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+            /* skippable frame */
             if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
-            memset(fparamsPtr, 0, sizeof(*fparamsPtr));
-            fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
-            fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
+            memset(fhiPtr, 0, sizeof(*fhiPtr));
+            fhiPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
+            fhiPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
             return 0;
         }
         return ERROR(prefix_unknown);
@@ -281,10 +282,10 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* fparamsPtr, const void* src, size_t
         }
         if (!windowSize) windowSize = (U32)frameContentSize;
         if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
-        fparamsPtr->frameContentSize = frameContentSize;
-        fparamsPtr->windowSize = windowSize;
-        fparamsPtr->dictID = dictID;
-        fparamsPtr->checksumFlag = checksumFlag;
+        fhiPtr->frameContentSize = frameContentSize;
+        fhiPtr->windowSize = windowSize;
+        fhiPtr->dictID = dictID;
+        fhiPtr->checksumFlag = checksumFlag;
     }
     return 0;
 }
@@ -2182,6 +2183,15 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
            + zds->inBuffSize + zds->outBuffSize;
 }
 
+size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader)
+{
+    size_t const windowSize = fHeader.windowSize;
+    size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
+    size_t const inBuffSize = blockSize;  /* no block can be larger */
+    size_t const outBuffSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2);
+    return sizeof(ZSTD_DStream) + ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
+}
+
 
 /* *****   Decompression   ***** */
 
diff --git a/lib/zstd.h b/lib/zstd.h
index ddcaaf8bd..08754cb2a 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -398,6 +398,13 @@ typedef struct {
     ZSTD_frameParameters fParams;
 } ZSTD_parameters;
 
+typedef struct {
+    unsigned long long frameContentSize;
+    unsigned windowSize;
+    unsigned dictID;
+    unsigned checksumFlag;
+} ZSTD_frameHeader;
+
 /*= Custom memory allocation functions */
 typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
@@ -479,6 +486,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
  *         an internal ?Dict will be created, which size is not estimated.
  *         In this case, get additional size by using ZSTD_estimate?DictSize */
 ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
+ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
 /*! ZSTD_estimate?DictSize() :
  *  Note : if dictionary is created "byReference", reduce estimation by dictSize */
@@ -741,13 +749,6 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   It also returns Frame Size as fparamsPtr->frameContentSize.
 */
 
-typedef struct {
-    unsigned long long frameContentSize;
-    unsigned windowSize;
-    unsigned dictID;
-    unsigned checksumFlag;
-} ZSTD_frameHeader;
-
 /*=====   Buffer-less streaming decompression functions  =====*/
 ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
 ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 6c210f2ff..6ce309c55 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -262,7 +262,20 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }   }
     DISPLAYLEVEL(3, "OK \n");
 
-    DISPLAYLEVEL(3, "test%3i : check DStream size : ", testNb++);
+    /* context size functions */
+    DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
+    {   ZSTD_frameHeader fhi;
+        const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
+        size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
+        if (gfhError!=0) goto _output_error;
+        DISPLAYLEVEL(5, " (windowSize : %u) ", fhi.windowSize);
+        {   size_t const s = ZSTD_estimateDStreamSize(fhi)
+                           + ZSTD_estimateDDictSize(128 KB);  /* uses ZSTD_initDStream_usingDict() */
+            if (ZSTD_isError(s)) goto _output_error;
+            DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);
+    }   }
+
+    DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
     { size_t const s = ZSTD_sizeof_DStream(zd);
       if (ZSTD_isError(s)) goto _output_error;
       DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s);