Create a ZSTD compression context using external alloc and free functions
ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +workspace: The memory area to emplace the context into. + Provided pointer must 8-bytes aligned. + It must outlive context usage. + workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize() + to determine how large workspace must be to support scenario. + @return : pointer to ZSTD_CCtx*, or NULL if error (size too small) + Note : zstd will never resize nor malloc() when using a static cctx. + If it needs more memory than available, it will simply error out. + Note 2 : there is no corresponding "free" function. + Since workspace was allocated externally, it must be freed externally too. + Limitation : currently not compatible with internal CDict creation, such as + ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict(). + +
typedef enum {
ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */
@@ -714,7 +730,7 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);
/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 0bed04fb4..deee3cfc7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -126,6 +126,7 @@ struct ZSTD_CCtx_s {
U64 consumedSrcSize;
XXH64_state_t xxhState;
ZSTD_customMem customMem;
+ size_t staticSize;
seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable;
@@ -175,9 +176,38 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
return cctx;
}
+ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
+{
+ ZSTD_CCtx* cctx = (ZSTD_CCtx*) workspace;
+ if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */
+ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */
+ memset(workspace, 0, workspaceSize);
+ cctx->staticSize = workspaceSize;
+ cctx->workSpace = (void*)(cctx+1);
+ cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx);
+
+ /* entropy space (never moves) */
+ /* note : this code should be shared with resetCCtx, instead of copied */
+ { void* ptr = cctx->workSpace;
+ cctx->hufCTable = (HUF_CElt*)ptr;
+ ptr = (char*)cctx->hufCTable + hufCTable_size; /* note : HUF_CElt* is incomplete type, size is estimated via macro */
+ cctx->offcodeCTable = (FSE_CTable*) ptr;
+ ptr = (char*)ptr + offcodeCTable_size;
+ cctx->matchlengthCTable = (FSE_CTable*) ptr;
+ ptr = (char*)ptr + matchlengthCTable_size;
+ cctx->litlengthCTable = (FSE_CTable*) ptr;
+ ptr = (char*)ptr + litlengthCTable_size;
+ assert(((size_t)ptr & 3) == 0); /* ensure correct alignment */
+ cctx->entropyScratchSpace = (unsigned*) ptr;
+ }
+
+ return cctx;
+}
+
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
{
if (cctx==NULL) return 0; /* support free on NULL */
+ assert(!cctx->staticSize); /* not compatible with static CCtx */
ZSTD_free(cctx->workSpace, cctx->customMem);
cctx->workSpace = NULL;
ZSTD_freeCDict(cctx->cdictLocal);
@@ -337,6 +367,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
{
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
+ if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */
ZSTD_freeCDict(cctx->cdictLocal); /* in case one already exists */
if (dict==NULL || dictSize==0) { /* no dictionary mode */
cctx->cdictLocal = NULL;
@@ -448,6 +479,17 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
return sizeof(ZSTD_CCtx) + neededSpace;
}
+size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
+{
+ size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
+ size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
+ size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
+ size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
+ size_t const streamingSize = inBuffSize + outBuffSize;
+
+ return CCtxSize + streamingSize;
+}
+
static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
ZSTD_compressionParameters cParams2)
@@ -532,9 +574,14 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
buffInSize + buffOutSize : 0;
size_t const neededSpace = entropySpace + optSpace + tableSpace
+ tokenSpace + bufferSpace;
- if (zc->workSpaceSize < neededSpace) {
+
+ if (zc->workSpaceSize < neededSpace) { /* too small : resize /*/
DEBUGLOG(5, "Need to update workSpaceSize from %uK to %uK \n",
- (unsigned)zc->workSpaceSize>>10, (unsigned)neededSpace>>10);
+ (unsigned)zc->workSpaceSize>>10,
+ (unsigned)neededSpace>>10);
+ /* static cctx : no resize, error out */
+ if (zc->staticSize) return ERROR(memory_allocation);
+
zc->workSpaceSize = 0;
ZSTD_free(zc->workSpace, zc->customMem);
zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
@@ -3362,16 +3409,6 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
return ZSTD_freeCCtx(zcs); /* same object */
}
-size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
-{
- size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
- size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
- size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
- size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
- size_t const streamingSize = inBuffSize + outBuffSize;
-
- return CCtxSize + streamingSize;
-}
/*====== Initialization ======*/
@@ -3446,6 +3483,10 @@ static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
zcs->cdict = NULL;
if (dict && dictSize >= 8) {
+ if (zcs->staticSize) { /* static CCtx : never uses malloc */
+ /* incompatible with internal cdict creation */
+ return ERROR(memory_allocation);
+ }
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0 /* copy */, params.cParams, zcs->customMem);
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
diff --git a/lib/zstd.h b/lib/zstd.h
index 83fe9d30f..d25130faf 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -511,6 +511,22 @@ ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize);
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
+/*! ZSTD_initStaticCCtx() : initialize a fixed-size zstd compression context
+ * workspace: The memory area to emplace the context into.
+ * Provided pointer must 8-bytes aligned.
+ * It must outlive context usage.
+ * workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize()
+ * to determine how large workspace must be to support scenario.
+ * @return : pointer to ZSTD_CCtx*, or NULL if error (size too small)
+ * Note : zstd will never resize nor malloc() when using a static cctx.
+ * If it needs more memory than available, it will simply error out.
+ * Note 2 : there is no corresponding "free" function.
+ * Since workspace was allocated externally, it must be freed externally too.
+ * Limitation : currently not compatible with internal CDict creation, such as
+ * ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict().
+ */
+ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
+
/* !!! Soon to be deprecated !!! */
typedef enum {
@@ -841,7 +857,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
-ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */
+ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 583018a45..a1dc6ba77 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -190,6 +190,63 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n");
+ /* Static CCtx tests */
+#define STATIC_CCTX_LEVEL 3
+ DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
+ { ZSTD_compressionParameters const cParams = ZSTD_getCParams(STATIC_CCTX_LEVEL, 0, 0);
+ size_t const staticCCtxSize = ZSTD_estimateCStreamSize(cParams);
+ void* staticCCtxBuffer = malloc(staticCCtxSize);
+ if (staticCCtxBuffer==NULL) {
+ DISPLAY("Not enough memory, aborting\n");
+ testResult = 1;
+ goto _end;
+ }
+ { ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
+ if (staticCCtx==NULL) goto _output_error;
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
+ { size_t const r = ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL);
+ if (ZSTD_isError(r)) goto _output_error; }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : simple compression test with static CCtx : ", testNb++);
+ CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx,
+ compressedBuffer, ZSTD_compressBound(CNBuffSize),
+ CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL),
+ cSize=r );
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+ DISPLAYLEVEL(4, "test%3i : decompress verification test : ", testNb++);
+ { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
+ if (r != CNBuffSize) goto _output_error; }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
+ { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
+ if (!ZSTD_isError(r)) goto _output_error; }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
+ { size_t const r = ZSTD_compressBegin(staticCCtx, 1);
+ if (ZSTD_isError(r)) goto _output_error; }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : init CStream for small level %u : ", testNb++, 1);
+ { size_t const r = ZSTD_initCStream(staticCCtx, 1);
+ if (ZSTD_isError(r)) goto _output_error; }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : init CStream with dictionary (should fail) : ", testNb++);
+ { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
+ if (!ZSTD_isError(r)) goto _output_error; }
+ DISPLAYLEVEL(4, "OK \n");
+ }
+ free(staticCCtxBuffer);
+ }
+
+
+
/* ZSTDMT simple MT compression test */
DISPLAYLEVEL(4, "test%3i : create ZSTDMT CCtx : ", testNb++);
{ ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);