lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

finalized retrofit of ZSTD_CCtx_reset()

Browse Source 
updated all depending sources
This commit is contained in:
Yann Collet
2018-11-14 13:05:35 -08:00
parent 5adbad4059
commit 7b0391e37e
6 changed files with 440 additions and 422 deletions
Download Patch File Download Diff File Expand all files Collapse all files

845
doc/zstd_manual.html
View File

@ -21,16 +21,16 @@
<li><a href="#Chapter11">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li> <li><a href="#Chapter11">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
<li><a href="#Chapter12">Candidate API for promotion into stable</a></li> <li><a href="#Chapter12">Candidate API for promotion into stable</a></li>
<li><a href="#Chapter13">Memory management</a></li> <li><a href="#Chapter13">Memory management</a></li>
<li><a href="#Chapter14">Purely experimental API</a></li> <li><a href="#Chapter14">Advanced compression API</a></li>
<li><a href="#Chapter15">Frame size functions</a></li> <li><a href="#Chapter15">experimental API (static linking only)</a></li>
<li><a href="#Chapter16">Memory management</a></li> <li><a href="#Chapter16">Frame size functions</a></li>
<li><a href="#Chapter17">Advanced compression functions</a></li> <li><a href="#Chapter17">Memory management</a></li>
<li><a href="#Chapter18">Advanced decompression functions</a></li> <li><a href="#Chapter18">Advanced compression functions</a></li>
<li><a href="#Chapter19">Advanced streaming functions</a></li> <li><a href="#Chapter19">Advanced decompression functions</a></li>
<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li> <li><a href="#Chapter20">Advanced streaming functions</a></li>
<li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li> <li><a href="#Chapter21">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li> <li><a href="#Chapter22">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter23">New advanced API (experimental)</a></li> <li><a href="#Chapter23">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter24">Block level API</a></li> <li><a href="#Chapter24">Block level API</a></li>
</ol> </ol>
<hr> <hr>
@ -370,14 +370,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
Object memory usage can evolve when re-used. Object memory usage can evolve when re-used.
</p></pre><BR> </p></pre><BR>
<a name="Chapter14"></a><h2>Purely experimental API</h2><pre> <a name="Chapter14"></a><h2>Advanced compression API</h2><pre></pre>
The following symbols and constants
are not planned to join "stable API" status anytime soon.
Some will never reach "stable", and remain in the static_only section.
Some of them might be removed in the future.
<BR></pre>
<pre><b>typedef enum {
</b>/* Opened question : should we have a format ZSTD_f_auto ?<b>
* Today, it would mean exactly the same as ZSTD_f_zstd1.
* But, in the future, should several formats become supported,
* on the compression side, it would mean "default format".
* On the decompression side, it would mean "automatic format detection",
* so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
* Since meaning is a little different, another option could be to define different enums for compression and decompression.
* This question could be kept for later, when there are actually multiple formats to support,
* but there is also the question of pinning enum values, and pinning value `0` is especially important */
ZSTD_f_zstd1 = 0, </b>/* zstd frame format, specified in zstd_compression_format.md (default) */<b>
ZSTD_f_zstd1_magicless = 1, </b>/* Variant of zstd frame format, without initial 4-bytes magic number.<b>
* Useful to save 4 bytes per generated frame.
* Decoder cannot recognise automatically this format, requiring instructions. */
} ZSTD_format_e;
</b></pre><BR>
<pre><b>typedef enum { ZSTD_fast=1, <pre><b>typedef enum { ZSTD_fast=1,
ZSTD_dfast=2, ZSTD_dfast=2,
ZSTD_greedy=3, ZSTD_greedy=3,
@ -389,6 +399,325 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
</b>/* note : new strategies might be added in the future */<b> </b>/* note : new strategies might be added in the future */<b>
} ZSTD_strategy; } ZSTD_strategy;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef enum {
</b>/* Note: this enum and the behavior it controls are effectively internal<b>
* implementation details of the compressor. They are expected to continue
* to evolve and should be considered only in the context of extremely
* advanced performance tuning.
*
* Zstd currently supports the use of a CDict in two ways:
*
* - The contents of the CDict can be copied into the working context. This
* means that the compression can search both the dictionary and input
* while operating on a single set of internal tables. This makes
* the compression faster per-byte of input. However, the initial copy of
* the CDict's tables incurs a fixed cost at the beginning of the
* compression. For small compressions (< 8 KB), that copy can dominate
* the cost of the compression.
*
* - The CDict's tables can be used in-place. In this model, compression is
* slower per input byte, because the compressor has to search two sets of
* tables. However, this model incurs no start-up cost (as long as the
* working context's tables can be reused). For small inputs, this can be
* faster than copying the CDict's tables.
*
* Zstd has a simple internal heuristic that selects which strategy to use
* at the beginning of a compression. However, if experimentation shows that
* Zstd is making poor choices, it is possible to override that choice with
* this enum.
*/
ZSTD_dictDefaultAttach = 0, </b>/* Use the default heuristic. */<b>
ZSTD_dictForceAttach = 1, </b>/* Never copy the dictionary. */<b>
ZSTD_dictForceCopy = 2, </b>/* Always copy the dictionary. */<b>
} ZSTD_dictAttachPref_e;
</b></pre><BR>
<pre><b>typedef enum {
</b>/* compression parameters */<b>
ZSTD_p_compressionLevel=100, </b>/* Update all compression parameters according to pre-defined cLevel table<b>
* Default level is ZSTD_CLEVEL_DEFAULT==3.
* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
* Note 1 : it's possible to pass a negative compression level by casting it to unsigned type.
* Note 2 : setting a level sets all default values of other compression parameters.
* Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */
ZSTD_p_windowLog=101, </b>/* Maximum allowed back-reference distance, expressed as power of 2.<b>
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
* Special: value 0 means "use default windowLog".
* Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
* requires explicitly allowing such window size during decompression stage. */
ZSTD_p_hashLog=102, </b>/* Size of the initial probe table, as a power of 2.<b>
* Resulting table size is (1 << (hashLog+2)).
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
* Larger tables improve compression ratio of strategies <= dFast,
* and improve speed of strategies > dFast.
* Special: value 0 means "use default hashLog". */
ZSTD_p_chainLog=103, </b>/* Size of the multi-probe search table, as a power of 2.<b>
* Resulting table size is (1 << (chainLog+2)).
* Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
* Larger tables result in better and slower compression.
* This parameter is useless when using "fast" strategy.
* Note it's still useful when using "dfast" strategy,
* in which case it defines a secondary probe table.
* Special: value 0 means "use default chainLog". */
ZSTD_p_searchLog=104, </b>/* Number of search attempts, as a power of 2.<b>
* More attempts result in better and slower compression.
* This parameter is useless when using "fast" and "dFast" strategies.
* Special: value 0 means "use default searchLog". */
ZSTD_p_minMatch=105, </b>/* Minimum size of searched matches (note : repCode matches can be smaller).<b>
* Larger values make faster compression and decompression, but decrease ratio.
* Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
* Note that currently, for all strategies < btopt, effective minimum is 4.
* , for all strategies > fast, effective maximum is 6.
* Special: value 0 means "use default minMatchLength". */
ZSTD_p_targetLength=106, </b>/* Impact of this field depends on strategy.<b>
* For strategies btopt & btultra:
* Length of Match considered "good enough" to stop search.
* Larger values make compression stronger, and slower.
* For strategy fast:
* Distance between match sampling.
* Larger values make compression faster, and weaker.
* Special: value 0 means "use default targetLength". */
ZSTD_p_compressionStrategy=107, </b>/* See ZSTD_strategy enum definition.<b>
* Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "use default strategy". */
</b>/* LDM mode parameters */<b>
ZSTD_p_enableLongDistanceMatching=160, </b>/* Enable long distance matching.<b>
* This parameter is designed to improve compression ratio
* for large inputs, by finding large matches at long distance.
* It increases memory usage and window size.
* Note: enabling this parameter increases ZSTD_p_windowLog to 128 MB
* except when expressly set to a different value. */
ZSTD_p_ldmHashLog=161, </b>/* Size of the table for long distance matching, as a power of 2.<b>
* Larger values increase memory usage and compression ratio,
* but decrease compression speed.
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
* default: windowlog - 7.
* Special: value 0 means "automatically determine hashlog". */
ZSTD_p_ldmMinMatch=162, </b>/* Minimum match size for long distance matcher.<b>
* Larger/too small values usually decrease compression ratio.
* Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
* Special: value 0 means "use default value" (default: 64). */
ZSTD_p_ldmBucketSizeLog=163, </b>/* Log size of each bucket in the LDM hash table for collision resolution.<b>
* Larger values improve collision resolution but decrease compression speed.
* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX .
* Special: value 0 means "use default value" (default: 3). */
ZSTD_p_ldmHashEveryLog=164, </b>/* Frequency of inserting/looking up entries into the LDM hash table.<b>
* Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
* Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
* Larger values improve compression speed.
* Deviating far from default value will likely result in a compression ratio decrease.
* Special: value 0 means "automatically determine hashEveryLog". */
</b>/* frame parameters */<b>
ZSTD_p_contentSizeFlag=200, </b>/* Content size will be written into frame header _whenever known_ (default:1)<b>
* Content size must be known at the beginning of compression,
* it is provided using ZSTD_CCtx_setPledgedSrcSize() */
ZSTD_p_checksumFlag=201, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
ZSTD_p_dictIDFlag=202, </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>
</b>/* multi-threading parameters */<b>
</b>/* These parameters are only useful if multi-threading is enabled (ZSTD_MULTITHREAD).<b>
* They return an error otherwise. */
ZSTD_p_nbWorkers=400, </b>/* Select how many threads will be spawned to compress in parallel.<b>
* When nbWorkers >= 1, triggers asynchronous mode :
* ZSTD_compress_generic() consumes some input, flush some output if possible, and immediately gives back control to caller,
* while compression work is performed in parallel, within worker threads.
* (note : a strong exception to this rule is when first invocation sets ZSTD_e_end : it becomes a blocking call).
* More workers improve speed, but also increase memory usage.
* Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
ZSTD_p_jobSize=401, </b>/* Size of a compression job. This value is enforced only in non-blocking mode.<b>
* Each compression job is completed in parallel, so this value indirectly controls the nb of active threads.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 MB, whichever is largest.
* The minimum size is automatically and transparently enforced */
ZSTD_p_overlapSizeLog=402, </b>/* Size of previous input reloaded at the beginning of each job.<b>
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
</b>/* =================================================================== */<b>
</b>/* experimental parameters - no stability guaranteed */<b>
</b>/* note : should this be exported in a different section of zstd.h ? */<b>
</b>/* =================================================================== */<b>
</b>/* compression format */<b>
ZSTD_p_format = 10, </b>/* See ZSTD_format_e enum definition.<b>
* Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
ZSTD_p_forceMaxWindow=1100, </b>/* Force back-reference distances to remain < windowSize,<b>
* even when referencing into Dictionary content (default:0) */
ZSTD_p_forceAttachDict, </b>/* Controls whether the contents of a CDict are<b>
* used in place, or whether they are copied into
* the working context.
*
* Accepts values from the ZSTD_dictAttachPref_e
* enum. See the comments on that enum for an
* explanation of the feature.
*/
} ZSTD_cParameter;
</b></pre><BR>
<pre><b>size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
</b><p> Set one compression parameter, selected by enum ZSTD_cParameter.
Setting a parameter is generally only possible during frame initialization (before starting compression).
Exception : when using multi-threading mode (nbWorkers >= 1),
the following parameters can be updated _during_ compression (within same frame):
=> compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
new parameters will be active for next job only (after a flush()).
Note : when original `value` type is not unsigned (like int, or enum), cast it to unsigned.
@result : informational value (typically, value being effectively set, after clamping),
or an error code (which can be tested with ZSTD_isError()).
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned* value);
</b><p> Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
</b><p> Total input data size to be compressed as a single frame.
This value will be controlled at end of frame, and trigger an error if not respected.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Note 1 : 0 means zero, empty.
In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
Note 2 : If all data is provided and consumed in a single round,
this value is automatically overriden by srcSize instead.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
</b><p> Create an internal CDict from `dict` buffer.
Decompression will have to use same dictionary.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special: Adding a NULL (or 0-size) dictionary invalidates previous dictionary,
meaning "return to no-dictionary mode".
Note 1 : Dictionary will be used for all future compression jobs.
To return to "no-dictionary" situation, load a NULL dictionary
Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
For this reason, compression parameters cannot be changed anymore after loading a dictionary.
It's also a CPU consuming operation, with non-negligible impact on latency.
Note 3 :`dict` content will be copied internally.
Use ZSTD_CCtx_loadDictionary_byReference() to reference dictionary content instead.
In such a case, dictionary buffer must outlive its users.
Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
to precisely select how dictionary content must be interpreted.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
</b><p> Reference a prepared dictionary, to be used for all next compression jobs.
Note that compression parameters are enforced from within CDict,
and supercede any compression parameter previously set within CCtx.
The dictionary will remain valid for future compression jobs using same CCtx.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special : adding a NULL CDict means "return to no-dictionary mode".
Note 1 : Currently, only one dictionary can be managed.
Adding a new dictionary effectively "discards" any previous one.
Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
const void* prefix, size_t prefixSize);
</b><p> Reference a prefix (single-usage dictionary) for next compression job.
Decompression will need same prefix to properly regenerate data.
Compressing with a prefix is similar in outcome as performing a diff and compressing it,
but performs much faster, especially during decompression (compression speed is tunable with compression level).
Note that prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end).
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
Note 1 : Prefix buffer is referenced. It **must** outlive compression job.
Its contain must remain unmodified up to end of compression (ZSTD_e_end).
Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
ensure that the window size is large enough to contain the entire source.
See ZSTD_p_windowLog.
Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
It's a CPU consuming operation, with non-negligible impact on latency.
If there is a need to use same prefix multiple times, consider loadDictionary instead.
Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
Use ZSTD_CCtx_refPrefix_advanced() to alter dictionary intepretation.
</p></pre><BR>
<pre><b>typedef enum {
ZSTD_CCtx_reset_session_only = 1,
ZSTD_CCtx_reset_parameters = 2,
ZSTD_CCtx_reset_session_and_parameters = 3
} ZSTD_CCtx_reset_directive;
</b></pre><BR>
<pre><b>size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_CCtx_reset_directive zcrd);
</b><p> Return a CCtx to clean state.
There 2 things that can be reset, independently or jointly :
- The session : will stop compressing current frame, and stand ready to start a new one.
This action never fails.
Useful after an error, or to interrupt any ongoing compression.
Any internal data not yet flushed is cancelled.
But parameters and dictionary are kept unchanged.
Therefore, same parameters and dictionary will be used for next frame.
- The parameters : changes all parameters back to "default".
This removes any dictionary too.
Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
- Both : similar to resetting the session, followed by resetting parameters.
This action never fails.
</p></pre><BR>
<pre><b>typedef enum {
ZSTD_e_continue=0, </b>/* collect more data, encoder decides when to output compressed result, for optimal compression ratio */<b>
ZSTD_e_flush, </b>/* flush any data provided so far,<b>
* it creates (at least) one new block, that can be decoded immediately on reception;
* frame will continue: any future data can still reference previously compressed data, improving compression. */
ZSTD_e_end </b>/* flush any remaining data and close current frame.<b>
* any additional data starts a new frame.
* each frame is independent (does not reference any content from previous frame). */
} ZSTD_EndDirective;
</b></pre><BR>
<pre><b>size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp);
</b><p> Behave about the same as ZSTD_compressStream. To note :
- Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
- Compression parameters cannot be changed once compression is started.
- outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
- outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- In single-thread mode (default), function is blocking : it completed its job before returning to caller.
- In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
and then immediately returns, just indicating that there is some data remaining to be flushed.
The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
- @return provides a minimum amount of data remaining to be flushed from internal buffers
or an error code, which can be tested using ZSTD_isError().
if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
- after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
only ZSTD_e_end or ZSTD_e_flush operations are allowed.
Before starting a new compression job, or changing compression parameters,
it is required to fully flush internal buffers.
</p></pre><BR>
<pre><b>size_t ZSTD_compress_generic_simpleArgs (
ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, size_t* dstPos,
const void* src, size_t srcSize, size_t* srcPos,
ZSTD_EndDirective endOp);
</b><p> Same as ZSTD_compress_generic(),
but using only integral types as arguments.
Argument list is larger than ZSTD_{in,out}Buffer,
but can be helpful for binders from dynamic languages
which have troubles handling structures containing memory pointers.
</p></pre><BR>
<a name="Chapter15"></a><h2>experimental API (static linking only)</h2><pre>
The following symbols and constants
are not planned to join "stable API" status anytime soon.
Some will never reach "stable", and remain in the static_only section.
Some of them might be removed in the future.
<BR></pre>
<pre><b>typedef struct { <pre><b>typedef struct {
unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b> unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b>
unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b> unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b>
@ -421,7 +750,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
ZSTD_dlm_byRef = 1, </b>/**< Reference dictionary content -- the dictionary buffer must outlive its users. */<b> ZSTD_dlm_byRef = 1, </b>/**< Reference dictionary content -- the dictionary buffer must outlive its users. */<b>
} ZSTD_dictLoadMethod_e; } ZSTD_dictLoadMethod_e;
</b></pre><BR> </b></pre><BR>
<a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre> <a name="Chapter16"></a><h2>Frame size functions</h2><pre></pre>
<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); <pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
</b><p> `src` should point to the start of a ZSTD encoded frame or skippable frame </b><p> `src` should point to the start of a ZSTD encoded frame or skippable frame
@ -460,7 +789,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
or an error code (if srcSize is too small) or an error code (if srcSize is too small)
</p></pre><BR> </p></pre><BR>
<a name="Chapter16"></a><h2>Memory management</h2><pre></pre> <a name="Chapter17"></a><h2>Memory management</h2><pre></pre>
<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel); <pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@ -540,7 +869,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
</p></pre><BR> </p></pre><BR>
<a name="Chapter17"></a><h2>Advanced compression functions</h2><pre></pre> <a name="Chapter18"></a><h2>Advanced compression functions</h2><pre></pre>
<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); <pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
</b><p> Create a digested dictionary for compression </b><p> Create a digested dictionary for compression
@ -582,7 +911,86 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
</b><p> Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters </b><p> Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters
</p></pre><BR> </p></pre><BR>
<a name="Chapter18"></a><h2>Advanced decompression functions</h2><pre></pre> <pre><b>size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
</b><p> Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
It saves some memory, but also requires that `dict` outlives its usage within `cctx`
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
</b><p> Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
how to load the dictionary (by copy ? by reference ?)
and how to interpret it (automatic ? force raw mode ? full mode only ?)
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
</b><p> Same as ZSTD_CCtx_refPrefix(), but gives finer control over
and how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?)
</p></pre><BR>
<pre><b>ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</b><p> Quick howto :
- ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
- ZSTD_CCtxParam_setParameter() : Push parameters one by one into
an existing ZSTD_CCtx_params structure.
This is similar to
ZSTD_CCtx_setParameter().
- ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
an existing CCtx.
These parameters will be applied to
all subsequent compression jobs.
- ZSTD_compress_generic() : Do compression using the CCtx.
- ZSTD_freeCCtxParams() : Free the memory.
This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
for static allocation for single-threaded compression.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
</b><p> Reset params to default values.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
</b><p> Initializes the compression parameters of cctxParams according to
compression level. All other parameters are reset to their default values.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
</b><p> Initializes the compression and frame parameters of cctxParams according to
params. All other parameters are reset to their default values.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
</b><p> Similar to ZSTD_CCtx_setParameter.
Set one compression parameter, selected by enum ZSTD_cParameter.
Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
Note : when `value` is an enum, cast it to unsigned for proper type checking.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned* value);
</b><p> Similar to ZSTD_CCtx_getParameter.
Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_setParametersUsingCCtxParams(
ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
</b><p> Apply a set of ZSTD_CCtx_params to the compression context.
This can be done even after compression is started,
if nbWorkers==0, this will have no impact until a new compression is started.
if nbWorkers>=1, new parameters will be picked up at next job,
with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
</p></pre><BR>
<a name="Chapter19"></a><h2>Advanced decompression functions</h2><pre></pre>
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size); <pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier. </b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
@ -622,7 +1030,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code.
</p></pre><BR> </p></pre><BR>
<a name="Chapter19"></a><h2>Advanced streaming functions</h2><pre></pre> <a name="Chapter20"></a><h2>Advanced streaming functions</h2><pre></pre>
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */<b> <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */<b>
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b> size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b>
@ -671,14 +1079,14 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b> size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b>
size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b> size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
</pre></b><BR> </pre></b><BR>
<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> <a name="Chapter21"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
This is an advanced API, giving full control over buffer management, for users which need direct control over memory. This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
But it's also a complex one, with several restrictions, documented below. But it's also a complex one, with several restrictions, documented below.
Prefer normal streaming API for an easier experience. Prefer normal streaming API for an easier experience.
<BR></pre> <BR></pre>
<a name="Chapter21"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> <a name="Chapter22"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
A ZSTD_CCtx object is required to track streaming operations. A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations. ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@ -714,7 +1122,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
</pre></b><BR> </pre></b><BR>
<a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> <a name="Chapter23"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
A ZSTD_DCtx object is required to track streaming operations. A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times. A ZSTD_DCtx object can be re-used multiple times.
@ -805,393 +1213,6 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
</pre></b><BR> </pre></b><BR>
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
</b></pre><BR> </b></pre><BR>
<a name="Chapter23"></a><h2>New advanced API (experimental)</h2><pre></pre>
<pre><b>typedef enum {
</b>/* Opened question : should we have a format ZSTD_f_auto ?<b>
* Today, it would mean exactly the same as ZSTD_f_zstd1.
* But, in the future, should several formats become supported,
* on the compression side, it would mean "default format".
* On the decompression side, it would mean "automatic format detection",
* so that ZSTD_f_zstd1 would mean "accept *only* zstd frames".
* Since meaning is a little different, another option could be to define different enums for compression and decompression.
* This question could be kept for later, when there are actually multiple formats to support,
* but there is also the question of pinning enum values, and pinning value `0` is especially important */
ZSTD_f_zstd1 = 0, </b>/* zstd frame format, specified in zstd_compression_format.md (default) */<b>
ZSTD_f_zstd1_magicless, </b>/* Variant of zstd frame format, without initial 4-bytes magic number.<b>
* Useful to save 4 bytes per generated frame.
* Decoder cannot recognise automatically this format, requiring instructions. */
} ZSTD_format_e;
</b></pre><BR>
<pre><b>typedef enum {
</b>/* Note: this enum and the behavior it controls are effectively internal<b>
* implementation details of the compressor. They are expected to continue
* to evolve and should be considered only in the context of extremely
* advanced performance tuning.
*
* Zstd currently supports the use of a CDict in two ways:
*
* - The contents of the CDict can be copied into the working context. This
* means that the compression can search both the dictionary and input
* while operating on a single set of internal tables. This makes
* the compression faster per-byte of input. However, the initial copy of
* the CDict's tables incurs a fixed cost at the beginning of the
* compression. For small compressions (< 8 KB), that copy can dominate
* the cost of the compression.
*
* - The CDict's tables can be used in-place. In this model, compression is
* slower per input byte, because the compressor has to search two sets of
* tables. However, this model incurs no start-up cost (as long as the
* working context's tables can be reused). For small inputs, this can be
* faster than copying the CDict's tables.
*
* Zstd has a simple internal heuristic that selects which strategy to use
* at the beginning of a compression. However, if experimentation shows that
* Zstd is making poor choices, it is possible to override that choice with
* this enum.
*/
ZSTD_dictDefaultAttach = 0, </b>/* Use the default heuristic. */<b>
ZSTD_dictForceAttach = 1, </b>/* Never copy the dictionary. */<b>
ZSTD_dictForceCopy = 2, </b>/* Always copy the dictionary. */<b>
} ZSTD_dictAttachPref_e;
</b></pre><BR>
<pre><b>typedef enum {
</b>/* compression format */<b>
ZSTD_p_format = 10, </b>/* See ZSTD_format_e enum definition.<b>
* Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */
</b>/* compression parameters */<b>
ZSTD_p_compressionLevel=100, </b>/* Update all compression parameters according to pre-defined cLevel table<b>
* Default level is ZSTD_CLEVEL_DEFAULT==3.
* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
* Note 1 : it's possible to pass a negative compression level by casting it to unsigned type.
* Note 2 : setting a level sets all default values of other compression parameters.
* Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */
ZSTD_p_windowLog, </b>/* Maximum allowed back-reference distance, expressed as power of 2.<b>
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
* Special: value 0 means "use default windowLog".
* Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
* requires explicitly allowing such window size during decompression stage. */
ZSTD_p_hashLog, </b>/* Size of the initial probe table, as a power of 2.<b>
* Resulting table size is (1 << (hashLog+2)).
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
* Larger tables improve compression ratio of strategies <= dFast,
* and improve speed of strategies > dFast.
* Special: value 0 means "use default hashLog". */
ZSTD_p_chainLog, </b>/* Size of the multi-probe search table, as a power of 2.<b>
* Resulting table size is (1 << (chainLog+2)).
* Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
* Larger tables result in better and slower compression.
* This parameter is useless when using "fast" strategy.
* Note it's still useful when using "dfast" strategy,
* in which case it defines a secondary probe table.
* Special: value 0 means "use default chainLog". */
ZSTD_p_searchLog, </b>/* Number of search attempts, as a power of 2.<b>
* More attempts result in better and slower compression.
* This parameter is useless when using "fast" and "dFast" strategies.
* Special: value 0 means "use default searchLog". */
ZSTD_p_minMatch, </b>/* Minimum size of searched matches (note : repCode matches can be smaller).<b>
* Larger values make faster compression and decompression, but decrease ratio.
* Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX.
* Note that currently, for all strategies < btopt, effective minimum is 4.
* , for all strategies > fast, effective maximum is 6.
* Special: value 0 means "use default minMatchLength". */
ZSTD_p_targetLength, </b>/* Impact of this field depends on strategy.<b>
* For strategies btopt & btultra:
* Length of Match considered "good enough" to stop search.
* Larger values make compression stronger, and slower.
* For strategy fast:
* Distance between match sampling.
* Larger values make compression faster, and weaker.
* Special: value 0 means "use default targetLength". */
ZSTD_p_compressionStrategy, </b>/* See ZSTD_strategy enum definition.<b>
* Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility.
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "use default strategy". */
ZSTD_p_enableLongDistanceMatching=160, </b>/* Enable long distance matching.<b>
* This parameter is designed to improve compression ratio
* for large inputs, by finding large matches at long distance.
* It increases memory usage and window size.
* Note: enabling this parameter increases ZSTD_p_windowLog to 128 MB
* except when expressly set to a different value. */
ZSTD_p_ldmHashLog, </b>/* Size of the table for long distance matching, as a power of 2.<b>
* Larger values increase memory usage and compression ratio,
* but decrease compression speed.
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
* default: windowlog - 7.
* Special: value 0 means "automatically determine hashlog". */
ZSTD_p_ldmMinMatch, </b>/* Minimum match size for long distance matcher.<b>
* Larger/too small values usually decrease compression ratio.
* Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
* Special: value 0 means "use default value" (default: 64). */
ZSTD_p_ldmBucketSizeLog, </b>/* Log size of each bucket in the LDM hash table for collision resolution.<b>
* Larger values improve collision resolution but decrease compression speed.
* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX .
* Special: value 0 means "use default value" (default: 3). */
ZSTD_p_ldmHashEveryLog, </b>/* Frequency of inserting/looking up entries in the LDM hash table.<b>
* Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
* Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
* Larger values improve compression speed.
* Deviating far from default value will likely result in a compression ratio decrease.
* Special: value 0 means "automatically determine hashEveryLog". */
</b>/* frame parameters */<b>
ZSTD_p_contentSizeFlag=200, </b>/* Content size will be written into frame header _whenever known_ (default:1)<b>
* Content size must be known at the beginning of compression,
* it is provided using ZSTD_CCtx_setPledgedSrcSize() */
ZSTD_p_checksumFlag, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
ZSTD_p_dictIDFlag, </b>/* When applicable, dictionary's ID is written into frame header (default:1) */<b>
</b>/* multi-threading parameters */<b>
</b>/* These parameters are only useful if multi-threading is enabled (ZSTD_MULTITHREAD).<b>
* They return an error otherwise. */
ZSTD_p_nbWorkers=400, </b>/* Select how many threads will be spawned to compress in parallel.<b>
* When nbWorkers >= 1, triggers asynchronous mode :
* ZSTD_compress_generic() consumes some input, flush some output if possible, and immediately gives back control to caller,
* while compression work is performed in parallel, within worker threads.
* (note : a strong exception to this rule is when first invocation sets ZSTD_e_end : it becomes a blocking call).
* More workers improve speed, but also increase memory usage.
* Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */
ZSTD_p_jobSize, </b>/* Size of a compression job. This value is enforced only in non-blocking mode.<b>
* Each compression job is completed in parallel, so this value indirectly controls the nb of active threads.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlapSize, or 1 MB, whichever is largest.
* The minimum size is automatically and transparently enforced */
ZSTD_p_overlapSizeLog, </b>/* Size of previous input reloaded at the beginning of each job.<b>
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
</b>/* =================================================================== */<b>
</b>/* experimental parameters - no stability guaranteed */<b>
</b>/* =================================================================== */<b>
ZSTD_p_forceMaxWindow=1100, </b>/* Force back-reference distances to remain < windowSize,<b>
* even when referencing into Dictionary content (default:0) */
ZSTD_p_forceAttachDict, </b>/* Controls whether the contents of a CDict are<b>
* used in place, or whether they are copied into
* the working context.
*
* Accepts values from the ZSTD_dictAttachPref_e
* enum. See the comments on that enum for an
* explanation of the feature.
*/
} ZSTD_cParameter;
</b></pre><BR>
<pre><b>size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
</b><p> Set one compression parameter, selected by enum ZSTD_cParameter.
Setting a parameter is generally only possible during frame initialization (before starting compression).
Exception : when using multi-threading mode (nbThreads >= 1),
following parameters can be updated _during_ compression (within same frame):
=> compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
new parameters will be active on next job, or after a flush().
Note : when `value` type is not unsigned (int, or enum), cast it to unsigned for proper type checking.
@result : informational value (typically, value being set, correctly clamped),
or an error code (which can be tested with ZSTD_isError()).
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned* value);
</b><p> Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
</b><p> Total input data size to be compressed as a single frame.
This value will be controlled at the end, and result in error if not respected.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Note 1 : 0 means zero, empty.
In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job.
Note 2 : If all data is provided and consumed in a single round,
this value is overriden by srcSize instead.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
</b><p> Create an internal CDict from `dict` buffer.
Decompression will have to use same dictionary.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special: Adding a NULL (or 0-size) dictionary invalidates previous dictionary,
meaning "return to no-dictionary mode".
Note 1 : Dictionary will be used for all future compression jobs.
To return to "no-dictionary" situation, load a NULL dictionary
Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
For this reason, compression parameters cannot be changed anymore after loading a dictionary.
It's also a CPU consuming operation, with non-negligible impact on latency.
Note 3 :`dict` content will be copied internally.
Use ZSTD_CCtx_loadDictionary_byReference() to reference dictionary content instead.
In such a case, dictionary buffer must outlive its users.
Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
to precisely select how dictionary content must be interpreted.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
</b><p> Reference a prepared dictionary, to be used for all next compression jobs.
Note that compression parameters are enforced from within CDict,
and supercede any compression parameter previously set within CCtx.
The dictionary will remain valid for future compression jobs using same CCtx.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special : adding a NULL CDict means "return to no-dictionary mode".
Note 1 : Currently, only one dictionary can be managed.
Adding a new dictionary effectively "discards" any previous one.
Note 2 : CDict is just referenced, its lifetime must outlive CCtx.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
const void* prefix, size_t prefixSize);
size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx,
const void* prefix, size_t prefixSize,
ZSTD_dictContentType_e dictContentType);
</b><p> Reference a prefix (single-usage dictionary) for next compression job.
Decompression will need same prefix to properly regenerate data.
Compressing with a prefix is similar in outcome as performing a diff and compressing it,
but performs much faster, especially during decompression (compression speed is tunable with compression level).
Note that prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end).
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
Note 1 : Prefix buffer is referenced. It **must** outlive compression job.
Its contain must remain unmodified up to end of compression (ZSTD_e_end).
Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
ensure that the window size is large enough to contain the entire source.
See ZSTD_p_windowLog.
Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
It's a CPU consuming operation, with non-negligible impact on latency.
If there is a need to use same prefix multiple times, consider loadDictionary instead.
Note 4 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
</p></pre><BR>
<pre><b>void ZSTD_CCtx_reset(ZSTD_CCtx* cctx);
</b><p> Return a CCtx to clean state.
Useful after an error, or to interrupt an ongoing compression job and start a new one.
Any internal data not yet flushed is cancelled.
The parameters and dictionary are kept unchanged, to reset them use ZSTD_CCtx_resetParameters().
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx);
</b><p> All parameters are back to default values (compression level is ZSTD_CLEVEL_DEFAULT).
Dictionary (if any) is dropped.
Resetting parameters is only possible during frame initialization (before starting compression).
To reset the context use ZSTD_CCtx_reset().
@return 0 or an error code (which can be checked with ZSTD_isError()).
</p></pre><BR>
<pre><b>typedef enum {
ZSTD_e_continue=0, </b>/* collect more data, encoder decides when to output compressed result, for optimal compression ratio */<b>
ZSTD_e_flush, </b>/* flush any data provided so far,<b>
* it creates (at least) one new block, that can be decoded immediately on reception;
* frame will continue: any future data can still reference previously compressed data, improving compression. */
ZSTD_e_end </b>/* flush any remaining data and close current frame.<b>
* any additional data starts a new frame.
* each frame is independent (does not reference any content from previous frame). */
} ZSTD_EndDirective;
</b></pre><BR>
<pre><b>size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
ZSTD_outBuffer* output,
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp);
</b><p> Behave about the same as ZSTD_compressStream. To note :
- Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter()
- Compression parameters cannot be changed once compression is started.
- outpot->pos must be <= dstCapacity, input->pos must be <= srcSize
- outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
- In single-thread mode (default), function is blocking : it completed its job before returning to caller.
- In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads,
and then immediately returns, just indicating that there is some data remaining to be flushed.
The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
- Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller.
- @return provides a minimum amount of data remaining to be flushed from internal buffers
or an error code, which can be tested using ZSTD_isError().
if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
- after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
only ZSTD_e_end or ZSTD_e_flush operations are allowed.
Before starting a new compression job, or changing compression parameters,
it is required to fully flush internal buffers.
</p></pre><BR>
<pre><b>size_t ZSTD_compress_generic_simpleArgs (
ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, size_t* dstPos,
const void* src, size_t srcSize, size_t* srcPos,
ZSTD_EndDirective endOp);
</b><p> Same as ZSTD_compress_generic(),
but using only integral types as arguments.
Argument list is larger than ZSTD_{in,out}Buffer,
but can be helpful for binders from dynamic languages
which have troubles handling structures containing memory pointers.
</p></pre><BR>
<pre><b>ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</b><p> Quick howto :
- ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
- ZSTD_CCtxParam_setParameter() : Push parameters one by one into
an existing ZSTD_CCtx_params structure.
This is similar to
ZSTD_CCtx_setParameter().
- ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
an existing CCtx.
These parameters will be applied to
all subsequent compression jobs.
- ZSTD_compress_generic() : Do compression using the CCtx.
- ZSTD_freeCCtxParams() : Free the memory.
This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
for static allocation for single-threaded compression.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
</b><p> Reset params to default values.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
</b><p> Initializes the compression parameters of cctxParams according to
compression level. All other parameters are reset to their default values.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
</b><p> Initializes the compression and frame parameters of cctxParams according to
params. All other parameters are reset to their default values.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
</b><p> Similar to ZSTD_CCtx_setParameter.
Set one compression parameter, selected by enum ZSTD_cParameter.
Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams().
Note : when `value` is an enum, cast it to unsigned for proper type checking.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
</p></pre><BR>
<pre><b>size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned* value);
</b><p> Similar to ZSTD_CCtx_getParameter.
Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
@result : 0, or an error code (which can be tested with ZSTD_isError()).
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_setParametersUsingCCtxParams(
ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
</b><p> Apply a set of ZSTD_CCtx_params to the compression context.
This can be done even after compression is started,
if nbWorkers==0, this will have no impact until a new compression is started.
if nbWorkers>=1, new parameters will be picked up at next job,
with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
</p></pre><BR>
<h3>Advanced decompression API</h3><pre></pre><b><pre></b>/* ==================================== */<b> <h3>Advanced decompression API</h3><pre></pre><b><pre></b>/* ==================================== */<b>
</pre></b><BR> </pre></b><BR>
<pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); <pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);

2
lib/compress/zstd_compress.c
View File

@ -3862,7 +3862,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
{ size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); { size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
if ( ZSTD_isError(flushMin) if ( ZSTD_isError(flushMin)
|| (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
ZSTD_CCtx_reset(cctx); ZSTD_CCtx_reset(cctx, ZSTD_CCtx_reset_session_only);
} }
DEBUGLOG(5, "completed ZSTD_compress_generic delegating to ZSTDMT_compressStream_generic"); DEBUGLOG(5, "completed ZSTD_compress_generic delegating to ZSTDMT_compressStream_generic");
return flushMin; return flushMin;

3
programs/benchzstd.c
View File

@ -160,8 +160,7 @@ typedef struct {
static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, size_t dictBufferSize, int cLevel, const void* dictBuffer, size_t dictBufferSize, int cLevel,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
ZSTD_CCtx_reset(ctx); ZSTD_CCtx_reset(ctx, ZSTD_CCtx_reset_session_and_parameters);
ZSTD_CCtx_resetParameters(ctx);
if (adv->nbWorkers==1) { if (adv->nbWorkers==1) {
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
} else { } else {

4
tests/fuzzer.c
View File

@ -527,13 +527,13 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value)); CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
CHECK_EQ(value, ZSTD_HASHLOG_MIN); CHECK_EQ(value, ZSTD_HASHLOG_MIN);
/* Reset the CCtx */ /* Reset the CCtx */
ZSTD_CCtx_reset(cctx); ZSTD_CCtx_reset(cctx, ZSTD_CCtx_reset_session_only);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value)); CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
CHECK_EQ(value, 7); CHECK_EQ(value, 7);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value)); CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
CHECK_EQ(value, ZSTD_HASHLOG_MIN); CHECK_EQ(value, ZSTD_HASHLOG_MIN);
/* Reset the parameters */ /* Reset the parameters */
ZSTD_CCtx_resetParameters(cctx); ZSTD_CCtx_reset(cctx, ZSTD_CCtx_reset_parameters);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value)); CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
CHECK_EQ(value, 3); CHECK_EQ(value, 3);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value)); CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));

3
tests/paramgrill.c
View File

@ -893,8 +893,7 @@ typedef struct {
static size_t local_initCCtx(void* payload) { static size_t local_initCCtx(void* payload) {
const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
varInds_t i; varInds_t i;
ZSTD_CCtx_reset(ag->cctx); ZSTD_CCtx_reset(ag->cctx, ZSTD_CCtx_reset_session_and_parameters);
ZSTD_CCtx_resetParameters(ag->cctx);
ZSTD_CCtx_setParameter(ag->cctx, ZSTD_p_compressionLevel, ag->cLevel); ZSTD_CCtx_setParameter(ag->cctx, ZSTD_p_compressionLevel, ag->cLevel);
for(i = 0; i < NUM_PARAMS; i++) { for(i = 0; i < NUM_PARAMS; i++) {

5
tests/zstreamtest.c
View File

@ -930,7 +930,7 @@ static int basicUnitTests(U32 seed, double compressibility)
if (!cdict || !ddict) goto _output_error; if (!cdict || !ddict) goto _output_error;
ZSTD_CCtx_reset(zc); ZSTD_CCtx_reset(zc, ZSTD_CCtx_reset_session_only);
ZSTD_resetDStream(zd); ZSTD_resetDStream(zd);
CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict)); CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict)); CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
@ -1077,8 +1077,7 @@ static int basicUnitTests(U32 seed, double compressibility)
int remainingInput = 256 * 1024; int remainingInput = 256 * 1024;
int offset; int offset;
ZSTD_CCtx_reset(zc); CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_CCtx_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_resetParameters(zc));
CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict)); CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, 1));
/* Write a bunch of 6 byte blocks */ /* Write a bunch of 6 byte blocks */