From 96f0cde31a33f2abc9f782c3b802126a0d235ab3 Mon Sep 17 00:00:00 2001
From: Yann Collet
size_t ZSTD_estimateCCtxSize(int compressionLevel); -size_t ZSTD_estimateCCtxSize_advanced_usingCParams(ZSTD_compressionParameters cParams); -size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params); +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); size_t ZSTD_estimateDCtxSize(void);These functions make it possible to estimate memory usage of a future {D,C}Ctx, before its creation. ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one. It will also consider src size to be arbitrarily "large", which is worst case. - If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced_usingCParams() can provide a tighter estimation. - ZSTD_estimateCCtxSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - ZSTD_estimateCCtxSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1. + If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. + ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1. Note : CCtx estimation is only correct for single-threaded compression
size_t ZSTD_estimateCStreamSize(int compressionLevel); -size_t ZSTD_estimateCStreamSize_advanced_usingCParams(ZSTD_compressionParameters cParams); -size_t ZSTD_estimateCStreamSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params); +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); size_t ZSTD_estimateDStreamSize(size_t windowSize); size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. It will also consider src size to be arbitrarily "large", which is worst case. - If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced_usingCParams() can provide a tighter estimation. - ZSTD_estimateCStreamSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - ZSTD_estimateCStreamSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1. + If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1. Note : CStream estimation is only correct for single-threaded compression. ZSTD_DStream memory budget depends on window Size. This information can be passed manually, using ZSTD_estimateDStreamSize, @@ -436,8 +436,8 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
typedef enum { - ZSTD_dlm_byCopy = 0, /* Copy dictionary content internally. */ - ZSTD_dlm_byRef, /* Reference dictionary content -- the dictionary buffer must outlives its users. */ + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ } ZSTD_dictLoadMethod_e;
size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); @@ -656,8 +656,8 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression para
This is an advanced API, giving full control over buffer management, for users which need direct control over memory. - But it's also a complex one, with many restrictions (documented below). - Prefer using normal streaming API for an easier experience + But it's also a complex one, with several restrictions, documented below. + Prefer normal streaming API for an easier experience.@@ -673,8 +673,8 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression para Then, consume your input using ZSTD_compressContinue(). There are some important considerations to keep in mind when using this advanced function : - - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only. - - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks. + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. Worst case evaluation is provided by ZSTD_compressBound(). ZSTD_compressContinue() doesn't guarantee recover after a failed compression. @@ -685,9 +685,9 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression para Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. - Without last block mark, frames will be considered unfinished (corrupted) by decoders. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. - `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame. + `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); diff --git a/examples/streaming_memory_usage.c b/examples/streaming_memory_usage.c index b709f50bd..b056c2a59 100644 --- a/examples/streaming_memory_usage.c +++ b/examples/streaming_memory_usage.c @@ -85,7 +85,7 @@ int main(int argc, char const *argv[]) { return 1; } } - + size_t compressedSize; { ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 }; ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 }; @@ -133,7 +133,7 @@ int main(int argc, char const *argv[]) { size_t const cstreamSize = ZSTD_sizeof_CStream(cstream); size_t const cstreamEstimatedSize = wLog ? - ZSTD_estimateCStreamSize_advanced_usingCParams(params.cParams) : + ZSTD_estimateCStreamSize_usingCParams(params.cParams) : ZSTD_estimateCStreamSize(compressionLevel); size_t const dstreamSize = ZSTD_sizeof_DStream(dstream); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 88eb51dcb..abb27961b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -673,7 +673,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); } -size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params) +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) { /* Estimate CCtx size is supported for single-threaded compression only. */ if (params->nbThreads > 1) { return ERROR(GENERIC); } @@ -710,22 +710,22 @@ size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* pa } } -size_t ZSTD_estimateCCtxSize_advanced_usingCParams(ZSTD_compressionParameters cParams) +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) { ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); - return ZSTD_estimateCCtxSize_advanced_usingCCtxParams(¶ms); + return ZSTD_estimateCCtxSize_usingCCtxParams(¶ms); } size_t ZSTD_estimateCCtxSize(int compressionLevel) { ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); - return ZSTD_estimateCCtxSize_advanced_usingCParams(cParams); + return ZSTD_estimateCCtxSize_usingCParams(cParams); } -size_t ZSTD_estimateCStreamSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params) +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) { if (params->nbThreads > 1) { return ERROR(GENERIC); } - { size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_usingCCtxParams(params); + { size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog); size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize; size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; @@ -735,15 +735,15 @@ size_t ZSTD_estimateCStreamSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* } } -size_t ZSTD_estimateCStreamSize_advanced_usingCParams(ZSTD_compressionParameters cParams) +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) { ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); - return ZSTD_estimateCStreamSize_advanced_usingCCtxParams(¶ms); + return ZSTD_estimateCStreamSize_usingCCtxParams(¶ms); } size_t ZSTD_estimateCStreamSize(int compressionLevel) { ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); - return ZSTD_estimateCStreamSize_advanced_usingCParams(cParams); + return ZSTD_estimateCStreamSize_usingCParams(cParams); } static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1, @@ -2182,8 +2182,8 @@ size_t ZSTD_estimateCDictSize_advanced( { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict)); DEBUGLOG(5, "CCtx estimate : %u", - (U32)ZSTD_estimateCCtxSize_advanced_usingCParams(cParams)); - return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_advanced_usingCParams(cParams) + (U32)ZSTD_estimateCCtxSize_usingCParams(cParams)); + return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_usingCParams(cParams) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); } @@ -2308,7 +2308,7 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize, ZSTD_dictMode_e dictMode, ZSTD_compressionParameters cParams) { - size_t const cctxSize = ZSTD_estimateCCtxSize_advanced_usingCParams(cParams); + size_t const cctxSize = ZSTD_estimateCCtxSize_usingCParams(cParams); size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize) + cctxSize; ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; diff --git a/lib/zstd.h b/lib/zstd.h index 655fa29d3..047f905b7 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -499,21 +499,21 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * of a future {D,C}Ctx, before its creation. * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one. * It will also consider src size to be arbitrarily "large", which is worst case. - * If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCCtxSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCCtxSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1. + * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1. * Note : CCtx estimation is only correct for single-threaded compression */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); /*! ZSTD_estimateCStreamSize() : * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. * It will also consider src size to be arbitrarily "large", which is worst case. - * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCStreamSize_advanced_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCStreamSize_advanced_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1. * Note : CStream estimation is only correct for single-threaded compression. * ZSTD_DStream memory budget depends on window Size. * This information can be passed manually, using ZSTD_estimateDStreamSize, @@ -522,14 +522,14 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); * an internal ?Dict will be created, which additional size is not estimated here. * In this case, get total size by adding ZSTD_estimate?DictSize */ ZSTDLIB_API size_t ZSTD_estimateCStreamSize(int compressionLevel); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced_usingCParams(ZSTD_compressionParameters cParams); -ZSTDLIB_API size_t ZSTD_estimateCStreamSize_advanced_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); ZSTDLIB_API size_t ZSTD_estimateDStreamSize(size_t windowSize); ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); typedef enum { - ZSTD_dlm_byCopy = 0, /* Copy dictionary content internally. */ - ZSTD_dlm_byRef, /* Reference dictionary content -- the dictionary buffer must outlives its users. */ + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ } ZSTD_dictLoadMethod_e; /*! ZSTD_estimate?DictSize() : @@ -760,8 +760,8 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompress * Buffer-less and synchronous inner streaming functions * * This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with many restrictions (documented below). -* Prefer using normal streaming API for an easier experience +* But it's also a complex one, with several restrictions, documented below. +* Prefer normal streaming API for an easier experience. ********************************************************************* */ /** @@ -778,8 +778,8 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompress Then, consume your input using ZSTD_compressContinue(). There are some important considerations to keep in mind when using this advanced function : - - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only. - - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks. + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. Worst case evaluation is provided by ZSTD_compressBound(). ZSTD_compressContinue() doesn't guarantee recover after a failed compression. @@ -790,9 +790,9 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompress Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. - Without last block mark, frames will be considered unfinished (corrupted) by decoders. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. - `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame. + `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. */ /*===== Buffer-less streaming compression functions =====*/ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 1bc48f401..317ec461c 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -391,8 +391,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_advanced_usingCParams(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_advanced_usingCParams(winners[cLevel].params); + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index e52335da0..613a879bf 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -200,11 +200,11 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* context size functions */ DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize); - size_t const s = ZSTD_estimateCStreamSize_advanced_usingCParams(cParams) - /* uses ZSTD_initCStream_usingDict() */ - + ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); - if (ZSTD_isError(s)) goto _output_error; - DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)s); + size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams); + size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */ + if (ZSTD_isError(cstreamSize)) goto _output_error; + if (ZSTD_isError(cdictSize)) goto _output_error; + DISPLAYLEVEL(3, "OK (%u bytes) \n", (U32)(cstreamSize + cdictSize)); } DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);