mirror of
https://github.com/facebook/zstd.git
synced 2025-07-30 22:23:13 +03:00
Merge pull request #2974 from facebook/fix2966_part3
Lazy parameters adaptation (part 1 - ZSTD_c_stableInBuffer)
This commit is contained in:
@ -38,6 +38,7 @@ const char* ERR_getErrorString(ERR_enum code)
|
|||||||
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
||||||
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
||||||
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
||||||
|
case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
|
||||||
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
||||||
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
|
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
|
||||||
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
|
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
|
||||||
|
@ -177,12 +177,9 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
|
|||||||
if (cctx==NULL) return 0; /* support free on NULL */
|
if (cctx==NULL) return 0; /* support free on NULL */
|
||||||
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
|
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
|
||||||
"not compatible with static CCtx");
|
"not compatible with static CCtx");
|
||||||
{
|
{ int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
|
||||||
int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
|
|
||||||
ZSTD_freeCCtxContent(cctx);
|
ZSTD_freeCCtxContent(cctx);
|
||||||
if (!cctxInWorkspace) {
|
if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem);
|
||||||
ZSTD_customFree(cctx, cctx->customMem);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@ -4486,6 +4483,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
|
|||||||
#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL)
|
#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL)
|
||||||
|
|
||||||
/*! ZSTD_compressBegin_internal() :
|
/*! ZSTD_compressBegin_internal() :
|
||||||
|
* Assumption : either @dict OR @cdict (or none) is non-NULL, never both
|
||||||
* @return : 0, or an error code */
|
* @return : 0, or an error code */
|
||||||
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
|
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
|
||||||
const void* dict, size_t dictSize,
|
const void* dict, size_t dictSize,
|
||||||
@ -4566,11 +4564,11 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
|
|||||||
&cctxParams, pledgedSrcSize);
|
&cctxParams, pledgedSrcSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
|
size_t
|
||||||
|
ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
|
||||||
{
|
{
|
||||||
ZSTD_CCtx_params cctxParams;
|
ZSTD_CCtx_params cctxParams;
|
||||||
{
|
{ ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
|
||||||
ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
|
|
||||||
ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
|
ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
|
||||||
}
|
}
|
||||||
DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
|
DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
|
||||||
@ -5317,30 +5315,41 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
|
|||||||
|
|
||||||
static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx)
|
static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx)
|
||||||
{
|
{
|
||||||
size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos;
|
if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
|
||||||
if (hintInSize==0) hintInSize = cctx->blockSize;
|
return cctx->blockSize - cctx->stableIn_notConsumed;
|
||||||
return hintInSize;
|
}
|
||||||
|
assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered);
|
||||||
|
{ size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos;
|
||||||
|
if (hintInSize==0) hintInSize = cctx->blockSize;
|
||||||
|
return hintInSize;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** ZSTD_compressStream_generic():
|
/** ZSTD_compressStream_generic():
|
||||||
* internal function for all *compressStream*() variants
|
* internal function for all *compressStream*() variants
|
||||||
* non-static, because can be called from zstdmt_compress.c
|
* @return : hint size for next input to complete ongoing block */
|
||||||
* @return : hint size for next input */
|
|
||||||
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
||||||
ZSTD_outBuffer* output,
|
ZSTD_outBuffer* output,
|
||||||
ZSTD_inBuffer* input,
|
ZSTD_inBuffer* input,
|
||||||
ZSTD_EndDirective const flushMode)
|
ZSTD_EndDirective const flushMode)
|
||||||
{
|
{
|
||||||
const char* const istart = (const char*)input->src;
|
const char* const istart = (assert(input != NULL), (const char*)input->src);
|
||||||
const char* const iend = input->size != 0 ? istart + input->size : istart;
|
const char* const iend = (istart != NULL) ? istart + input->size : istart;
|
||||||
const char* ip = input->pos != 0 ? istart + input->pos : istart;
|
const char* ip = (istart != NULL) ? istart + input->pos : istart;
|
||||||
char* const ostart = (char*)output->dst;
|
char* const ostart = (assert(output != NULL), (char*)output->dst);
|
||||||
char* const oend = output->size != 0 ? ostart + output->size : ostart;
|
char* const oend = (ostart != NULL) ? ostart + output->size : ostart;
|
||||||
char* op = output->pos != 0 ? ostart + output->pos : ostart;
|
char* op = (ostart != NULL) ? ostart + output->pos : ostart;
|
||||||
U32 someMoreWork = 1;
|
U32 someMoreWork = 1;
|
||||||
|
|
||||||
/* check expectations */
|
/* check expectations */
|
||||||
DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode);
|
DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos);
|
||||||
|
assert(zcs != NULL);
|
||||||
|
if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) {
|
||||||
|
assert(input->pos >= zcs->stableIn_notConsumed);
|
||||||
|
input->pos -= zcs->stableIn_notConsumed;
|
||||||
|
ip -= zcs->stableIn_notConsumed;
|
||||||
|
zcs->stableIn_notConsumed = 0;
|
||||||
|
}
|
||||||
if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
|
if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
|
||||||
assert(zcs->inBuff != NULL);
|
assert(zcs->inBuff != NULL);
|
||||||
assert(zcs->inBuffSize > 0);
|
assert(zcs->inBuffSize > 0);
|
||||||
@ -5349,8 +5358,10 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
|||||||
assert(zcs->outBuff != NULL);
|
assert(zcs->outBuff != NULL);
|
||||||
assert(zcs->outBuffSize > 0);
|
assert(zcs->outBuffSize > 0);
|
||||||
}
|
}
|
||||||
assert(output->pos <= output->size);
|
if (input->src == NULL) assert(input->size == 0);
|
||||||
assert(input->pos <= input->size);
|
assert(input->pos <= input->size);
|
||||||
|
if (output->dst == NULL) assert(output->size == 0);
|
||||||
|
assert(output->pos <= output->size);
|
||||||
assert((U32)flushMode <= (U32)ZSTD_e_end);
|
assert((U32)flushMode <= (U32)ZSTD_e_end);
|
||||||
|
|
||||||
while (someMoreWork) {
|
while (someMoreWork) {
|
||||||
@ -5382,8 +5393,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
|||||||
zcs->inBuff + zcs->inBuffPos, toLoad,
|
zcs->inBuff + zcs->inBuffPos, toLoad,
|
||||||
ip, iend-ip);
|
ip, iend-ip);
|
||||||
zcs->inBuffPos += loaded;
|
zcs->inBuffPos += loaded;
|
||||||
if (loaded != 0)
|
if (ip) ip += loaded;
|
||||||
ip += loaded;
|
|
||||||
if ( (flushMode == ZSTD_e_continue)
|
if ( (flushMode == ZSTD_e_continue)
|
||||||
&& (zcs->inBuffPos < zcs->inBuffTarget) ) {
|
&& (zcs->inBuffPos < zcs->inBuffTarget) ) {
|
||||||
/* not enough input to fill full block : stop here */
|
/* not enough input to fill full block : stop here */
|
||||||
@ -5394,6 +5404,20 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
|||||||
/* empty */
|
/* empty */
|
||||||
someMoreWork = 0; break;
|
someMoreWork = 0; break;
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
|
||||||
|
if ( (flushMode == ZSTD_e_continue)
|
||||||
|
&& ( (size_t)(iend - ip) < zcs->blockSize) ) {
|
||||||
|
/* can't compress a full block : stop here */
|
||||||
|
zcs->stableIn_notConsumed = (size_t)(iend - ip);
|
||||||
|
ip = iend; /* pretend to have consumed input */
|
||||||
|
someMoreWork = 0; break;
|
||||||
|
}
|
||||||
|
if ( (flushMode == ZSTD_e_flush)
|
||||||
|
&& (ip == iend) ) {
|
||||||
|
/* empty */
|
||||||
|
someMoreWork = 0; break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
/* compress current block (note : this stage cannot be stopped in the middle) */
|
/* compress current block (note : this stage cannot be stopped in the middle) */
|
||||||
DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
|
DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
|
||||||
@ -5401,9 +5425,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
|||||||
void* cDst;
|
void* cDst;
|
||||||
size_t cSize;
|
size_t cSize;
|
||||||
size_t oSize = oend-op;
|
size_t oSize = oend-op;
|
||||||
size_t const iSize = inputBuffered
|
size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress
|
||||||
? zcs->inBuffPos - zcs->inToCompress
|
: MIN((size_t)(iend - ip), zcs->blockSize);
|
||||||
: MIN((size_t)(iend - ip), zcs->blockSize);
|
|
||||||
if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable)
|
if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable)
|
||||||
cDst = op; /* compress into output buffer, to skip flush stage */
|
cDst = op; /* compress into output buffer, to skip flush stage */
|
||||||
else
|
else
|
||||||
@ -5426,19 +5449,16 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|
|||||||
if (!lastBlock)
|
if (!lastBlock)
|
||||||
assert(zcs->inBuffTarget <= zcs->inBuffSize);
|
assert(zcs->inBuffTarget <= zcs->inBuffSize);
|
||||||
zcs->inToCompress = zcs->inBuffPos;
|
zcs->inToCompress = zcs->inBuffPos;
|
||||||
} else {
|
} else { /* !inputBuffered, hence ZSTD_bm_stable */
|
||||||
unsigned const lastBlock = (ip + iSize == iend);
|
unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend);
|
||||||
assert(flushMode == ZSTD_e_end /* Already validated */);
|
|
||||||
cSize = lastBlock ?
|
cSize = lastBlock ?
|
||||||
ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) :
|
ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) :
|
||||||
ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize);
|
ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize);
|
||||||
/* Consume the input prior to error checking to mirror buffered mode. */
|
/* Consume the input prior to error checking to mirror buffered mode. */
|
||||||
if (iSize > 0)
|
if (ip) ip += iSize;
|
||||||
ip += iSize;
|
|
||||||
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
|
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
|
||||||
zcs->frameEnded = lastBlock;
|
zcs->frameEnded = lastBlock;
|
||||||
if (lastBlock)
|
if (lastBlock) assert(ip == iend);
|
||||||
assert(ip == iend);
|
|
||||||
}
|
}
|
||||||
if (cDst == op) { /* no need to flush */
|
if (cDst == op) { /* no need to flush */
|
||||||
op += cSize;
|
op += cSize;
|
||||||
@ -5514,8 +5534,10 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf
|
|||||||
/* After a compression call set the expected input/output buffer.
|
/* After a compression call set the expected input/output buffer.
|
||||||
* This is validated at the start of the next compression call.
|
* This is validated at the start of the next compression call.
|
||||||
*/
|
*/
|
||||||
static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input)
|
static void
|
||||||
|
ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input)
|
||||||
{
|
{
|
||||||
|
DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)");
|
||||||
if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
|
if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
|
||||||
cctx->expectedInBuffer = *input;
|
cctx->expectedInBuffer = *input;
|
||||||
}
|
}
|
||||||
@ -5534,22 +5556,22 @@ static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx,
|
|||||||
{
|
{
|
||||||
if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
|
if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
|
||||||
ZSTD_inBuffer const expect = cctx->expectedInBuffer;
|
ZSTD_inBuffer const expect = cctx->expectedInBuffer;
|
||||||
if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size)
|
if (expect.src != input->src || expect.pos != input->pos)
|
||||||
RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!");
|
RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!");
|
||||||
if (endOp != ZSTD_e_end)
|
|
||||||
RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!");
|
|
||||||
}
|
}
|
||||||
|
(void)endOp;
|
||||||
if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) {
|
if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) {
|
||||||
size_t const outBufferSize = output->size - output->pos;
|
size_t const outBufferSize = output->size - output->pos;
|
||||||
if (cctx->expectedOutBufferSize != outBufferSize)
|
if (cctx->expectedOutBufferSize != outBufferSize)
|
||||||
RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!");
|
RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!");
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
|
static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
|
||||||
ZSTD_EndDirective endOp,
|
ZSTD_EndDirective endOp,
|
||||||
size_t inSize) {
|
size_t inSize)
|
||||||
|
{
|
||||||
ZSTD_CCtx_params params = cctx->requestedParams;
|
ZSTD_CCtx_params params = cctx->requestedParams;
|
||||||
ZSTD_prefixDict const prefixDict = cctx->prefixDict;
|
ZSTD_prefixDict const prefixDict = cctx->prefixDict;
|
||||||
FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */
|
FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */
|
||||||
@ -5563,9 +5585,9 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
|
|||||||
params.compressionLevel = cctx->cdict->compressionLevel;
|
params.compressionLevel = cctx->cdict->compressionLevel;
|
||||||
}
|
}
|
||||||
DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
|
DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
|
||||||
if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-fix pledgedSrcSize */
|
if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */
|
||||||
{
|
|
||||||
size_t const dictSize = prefixDict.dict
|
{ size_t const dictSize = prefixDict.dict
|
||||||
? prefixDict.dictSize
|
? prefixDict.dictSize
|
||||||
: (cctx->cdict ? cctx->cdict->dictContentSize : 0);
|
: (cctx->cdict ? cctx->cdict->dictContentSize : 0);
|
||||||
ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1);
|
ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1);
|
||||||
@ -5606,7 +5628,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
|
|||||||
cctx->streamStage = zcss_load;
|
cctx->streamStage = zcss_load;
|
||||||
cctx->appliedParams = params;
|
cctx->appliedParams = params;
|
||||||
} else
|
} else
|
||||||
#endif
|
#endif /* ZSTD_MULTITHREAD */
|
||||||
{ U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1;
|
{ U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1;
|
||||||
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
|
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
|
||||||
FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
|
FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
|
||||||
@ -5632,6 +5654,8 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* @return provides a minimum amount of data remaining to be flushed from internal buffers
|
||||||
|
*/
|
||||||
size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
|
size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
|
||||||
ZSTD_outBuffer* output,
|
ZSTD_outBuffer* output,
|
||||||
ZSTD_inBuffer* input,
|
ZSTD_inBuffer* input,
|
||||||
@ -5646,8 +5670,27 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
|
|||||||
|
|
||||||
/* transparent initialization stage */
|
/* transparent initialization stage */
|
||||||
if (cctx->streamStage == zcss_init) {
|
if (cctx->streamStage == zcss_init) {
|
||||||
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed");
|
size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */
|
||||||
ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */
|
size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed;
|
||||||
|
if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */
|
||||||
|
&& (endOp == ZSTD_e_continue) /* no flush requested, more input to come */
|
||||||
|
&& (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */
|
||||||
|
if (cctx->stableIn_notConsumed) { /* not the first time */
|
||||||
|
/* check stable source guarantees */
|
||||||
|
RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer");
|
||||||
|
RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos");
|
||||||
|
}
|
||||||
|
/* pretend input was consumed, to give a sense forward progress */
|
||||||
|
input->pos = input->size;
|
||||||
|
/* save stable inBuffer, for later control, and flush/end */
|
||||||
|
cctx->expectedInBuffer = *input;
|
||||||
|
/* but actually input wasn't consumed, so keep track of position from where compression shall resume */
|
||||||
|
cctx->stableIn_notConsumed += inputSize;
|
||||||
|
/* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */
|
||||||
|
return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */
|
||||||
|
}
|
||||||
|
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed");
|
||||||
|
ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */
|
||||||
}
|
}
|
||||||
/* end of transparent initialization stage */
|
/* end of transparent initialization stage */
|
||||||
|
|
||||||
@ -5660,6 +5703,13 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
|
|||||||
ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams);
|
ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams);
|
||||||
cctx->cParamsChanged = 0;
|
cctx->cParamsChanged = 0;
|
||||||
}
|
}
|
||||||
|
if (cctx->stableIn_notConsumed) {
|
||||||
|
assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable);
|
||||||
|
/* some early data was skipped - make it available for consumption */
|
||||||
|
assert(input->pos >= cctx->stableIn_notConsumed);
|
||||||
|
input->pos -= cctx->stableIn_notConsumed;
|
||||||
|
cctx->stableIn_notConsumed = 0;
|
||||||
|
}
|
||||||
for (;;) {
|
for (;;) {
|
||||||
size_t const ipos = input->pos;
|
size_t const ipos = input->pos;
|
||||||
size_t const opos = output->pos;
|
size_t const opos = output->pos;
|
||||||
@ -5698,7 +5748,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
|
|||||||
ZSTD_setBufferExpectations(cctx, output, input);
|
ZSTD_setBufferExpectations(cctx, output, input);
|
||||||
return flushMin;
|
return flushMin;
|
||||||
}
|
}
|
||||||
#endif
|
#endif /* ZSTD_MULTITHREAD */
|
||||||
FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , "");
|
FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , "");
|
||||||
DEBUGLOG(5, "completed ZSTD_compressStream2");
|
DEBUGLOG(5, "completed ZSTD_compressStream2");
|
||||||
ZSTD_setBufferExpectations(cctx, output, input);
|
ZSTD_setBufferExpectations(cctx, output, input);
|
||||||
@ -6214,20 +6264,28 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
|
|||||||
|
|
||||||
/*====== Finalize ======*/
|
/*====== Finalize ======*/
|
||||||
|
|
||||||
|
static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs)
|
||||||
|
{
|
||||||
|
const ZSTD_inBuffer nullInput = { NULL, 0, 0 };
|
||||||
|
const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
|
||||||
|
return stableInput ? zcs->expectedInBuffer : nullInput;
|
||||||
|
}
|
||||||
|
|
||||||
/*! ZSTD_flushStream() :
|
/*! ZSTD_flushStream() :
|
||||||
* @return : amount of data remaining to flush */
|
* @return : amount of data remaining to flush */
|
||||||
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
|
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
|
||||||
{
|
{
|
||||||
ZSTD_inBuffer input = { NULL, 0, 0 };
|
ZSTD_inBuffer input = inBuffer_forEndFlush(zcs);
|
||||||
|
input.size = input.pos; /* do not ingest more input during flush */
|
||||||
return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush);
|
return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
|
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
|
||||||
{
|
{
|
||||||
ZSTD_inBuffer input = { NULL, 0, 0 };
|
ZSTD_inBuffer input = inBuffer_forEndFlush(zcs);
|
||||||
size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
|
size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
|
||||||
FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed");
|
FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed");
|
||||||
if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */
|
if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */
|
||||||
/* single thread mode : attempt to calculate remaining to flush more precisely */
|
/* single thread mode : attempt to calculate remaining to flush more precisely */
|
||||||
{ size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
|
{ size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
|
||||||
|
@ -410,6 +410,7 @@ struct ZSTD_CCtx_s {
|
|||||||
|
|
||||||
/* Stable in/out buffer verification */
|
/* Stable in/out buffer verification */
|
||||||
ZSTD_inBuffer expectedInBuffer;
|
ZSTD_inBuffer expectedInBuffer;
|
||||||
|
size_t stableIn_notConsumed; /* nb bytes within stable input buffer that are said to be consumed but are not */
|
||||||
size_t expectedOutBufferSize;
|
size_t expectedOutBufferSize;
|
||||||
|
|
||||||
/* Dictionary */
|
/* Dictionary */
|
||||||
|
30
lib/zstd.h
30
lib/zstd.h
@ -1834,13 +1834,16 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
|
|||||||
* Experimental parameter.
|
* Experimental parameter.
|
||||||
* Default is 0 == disabled. Set to 1 to enable.
|
* Default is 0 == disabled. Set to 1 to enable.
|
||||||
*
|
*
|
||||||
* Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
|
* Tells the compressor that input data presented with ZSTD_inBuffer
|
||||||
* between calls, except for the modifications that zstd makes to pos (the
|
* will ALWAYS be the same between calls.
|
||||||
* caller must not modify pos). This is checked by the compressor, and
|
* Technically, the @src pointer must never be changed,
|
||||||
* compression will fail if it ever changes. This means the only flush
|
* and the @pos field can only be updated by zstd.
|
||||||
* mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
|
* However, it's possible to increase the @size field,
|
||||||
* is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
|
* allowing scenarios where more data can be appended after compressions starts.
|
||||||
* MUST not be modified during compression or you will get data corruption.
|
* These conditions are checked by the compressor,
|
||||||
|
* and compression will fail if they are not respected.
|
||||||
|
* Also, data in the ZSTD_inBuffer within the range [src, src + pos)
|
||||||
|
* MUST not be modified during compression or it will result in data corruption.
|
||||||
*
|
*
|
||||||
* When this flag is enabled zstd won't allocate an input window buffer,
|
* When this flag is enabled zstd won't allocate an input window buffer,
|
||||||
* because the user guarantees it can reference the ZSTD_inBuffer until
|
* because the user guarantees it can reference the ZSTD_inBuffer until
|
||||||
@ -1848,18 +1851,15 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
|
|||||||
* large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
|
* large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
|
||||||
* avoid the memcpy() from the input buffer to the input window buffer.
|
* avoid the memcpy() from the input buffer to the input window buffer.
|
||||||
*
|
*
|
||||||
* NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
|
|
||||||
* That means this flag cannot be used with ZSTD_compressStream().
|
|
||||||
*
|
|
||||||
* NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
|
* NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
|
||||||
* this flag is ALWAYS memory safe, and will never access out-of-bounds
|
* this flag is ALWAYS memory safe, and will never access out-of-bounds
|
||||||
* memory. However, compression WILL fail if you violate the preconditions.
|
* memory. However, compression WILL fail if conditions are not respected.
|
||||||
*
|
*
|
||||||
* WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
|
* WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
|
||||||
* not be modified during compression or you will get data corruption. This
|
* not be modified during compression or it will result in data corruption.
|
||||||
* is because zstd needs to reference data in the ZSTD_inBuffer to find
|
* This is because zstd needs to reference data in the ZSTD_inBuffer to find
|
||||||
* matches. Normally zstd maintains its own window buffer for this purpose,
|
* matches. Normally zstd maintains its own window buffer for this purpose,
|
||||||
* but passing this flag tells zstd to use the user provided buffer.
|
* but passing this flag tells zstd to rely on user provided buffer instead.
|
||||||
*/
|
*/
|
||||||
#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
|
#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
|
||||||
|
|
||||||
|
@ -66,6 +66,7 @@ typedef enum {
|
|||||||
ZSTD_error_tableLog_tooLarge = 44,
|
ZSTD_error_tableLog_tooLarge = 44,
|
||||||
ZSTD_error_maxSymbolValue_tooLarge = 46,
|
ZSTD_error_maxSymbolValue_tooLarge = 46,
|
||||||
ZSTD_error_maxSymbolValue_tooSmall = 48,
|
ZSTD_error_maxSymbolValue_tooSmall = 48,
|
||||||
|
ZSTD_error_stabilityCondition_notRespected = 50,
|
||||||
ZSTD_error_stage_wrong = 60,
|
ZSTD_error_stage_wrong = 60,
|
||||||
ZSTD_error_init_missing = 62,
|
ZSTD_error_init_missing = 62,
|
||||||
ZSTD_error_memory_allocation = 64,
|
ZSTD_error_memory_allocation = 64,
|
||||||
|
@ -1206,7 +1206,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
|||||||
|
|
||||||
DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
|
DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
|
||||||
{ int level = -1;
|
{ int level = -1;
|
||||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||||
if (!cctx) goto _output_error;
|
if (!cctx) goto _output_error;
|
||||||
for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
|
for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
|
||||||
CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
|
CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
|
||||||
|
@ -613,7 +613,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
|
DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
|
||||||
} }
|
} }
|
||||||
|
|
||||||
/* Complex context re-use scenario */
|
/* Compression state re-use scenario */
|
||||||
DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
|
DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
|
||||||
ZSTD_freeCStream(zc);
|
ZSTD_freeCStream(zc);
|
||||||
zc = ZSTD_createCStream();
|
zc = ZSTD_createCStream();
|
||||||
@ -634,8 +634,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
||||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||||
DISPLAYLEVEL(5, "end1 ");
|
DISPLAYLEVEL(5, "end1 ");
|
||||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
|
||||||
if (r != 0) goto _output_error; } /* error, or some data not flushed */
|
|
||||||
}
|
}
|
||||||
/* use 2 */
|
/* use 2 */
|
||||||
{ size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
|
{ size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
|
||||||
@ -653,8 +652,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
|
||||||
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
|
||||||
DISPLAYLEVEL(5, "end2 ");
|
DISPLAYLEVEL(5, "end2 ");
|
||||||
{ size_t const r = ZSTD_endStream(zc, &outBuff);
|
if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
|
||||||
if (r != 0) goto _output_error; } /* error, or some data not flushed */
|
|
||||||
}
|
}
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
@ -771,11 +769,12 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Compression with ZSTD_c_stable{In,Out}Buffer */
|
/* Compression with ZSTD_c_stable{In,Out}Buffer */
|
||||||
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||||
ZSTD_inBuffer in;
|
ZSTD_inBuffer in;
|
||||||
ZSTD_outBuffer out;
|
ZSTD_outBuffer out;
|
||||||
size_t cctxSize1;
|
size_t cctxSize1;
|
||||||
size_t cctxSize2;
|
size_t cctxSize2;
|
||||||
|
assert(cctx != NULL);
|
||||||
in.src = CNBuffer;
|
in.src = CNBuffer;
|
||||||
in.size = CNBufferSize;
|
in.size = CNBufferSize;
|
||||||
out.dst = compressedBuffer;
|
out.dst = compressedBuffer;
|
||||||
@ -786,30 +785,33 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
|
CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
|
||||||
CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
|
CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
|
||||||
CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
|
CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
|
||||||
{ ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
|
/* @cctxSize2 : sizeof_CCtx when doing full streaming (no stable in/out) */
|
||||||
|
{ ZSTD_CCtx* const cctx2 = ZSTD_createCCtx();
|
||||||
|
assert(cctx2 != NULL);
|
||||||
in.pos = out.pos = 0;
|
in.pos = out.pos = 0;
|
||||||
CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
|
CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
|
||||||
CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
|
CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
|
||||||
CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
|
CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
|
||||||
ZSTD_freeCCtx(cctx2);
|
ZSTD_freeCCtx(cctx2);
|
||||||
}
|
}
|
||||||
{ ZSTD_CCtx* cctx3 = ZSTD_createCCtx();
|
/* @cctxSize1 : sizeof_CCtx when doing single-shot compression (no streaming) */
|
||||||
|
{ ZSTD_CCtx* const cctx1 = ZSTD_createCCtx();
|
||||||
ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
|
ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
|
||||||
size_t cSize3;
|
size_t cSize3;
|
||||||
|
assert(cctx1 != NULL);
|
||||||
params.fParams.checksumFlag = 1;
|
params.fParams.checksumFlag = 1;
|
||||||
cSize3 = ZSTD_compress_advanced(cctx3, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
|
cSize3 = ZSTD_compress_advanced(cctx1, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
|
||||||
CHECK_Z(cSize3);
|
CHECK_Z(cSize3);
|
||||||
CHECK(!(cSize == cSize3), "Must be same compressed size");
|
CHECK(!(cSize == cSize3), "Must be same compressed size");
|
||||||
CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx3)), "Must be same CCtx size");
|
CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx1)), "Must be same CCtx size");
|
||||||
ZSTD_freeCCtx(cctx3);
|
ZSTD_freeCCtx(cctx1);
|
||||||
}
|
}
|
||||||
CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
|
CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
|
||||||
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
|
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
|
DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
|
||||||
{
|
{ int stableInBuffer;
|
||||||
int stableInBuffer;
|
|
||||||
int stableOutBuffer;
|
int stableOutBuffer;
|
||||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
|
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
|
||||||
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
|
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
|
||||||
@ -866,30 +868,73 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
in.pos = 0;
|
in.pos = 0;
|
||||||
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
|
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
|
||||||
CHECK(!ZSTD_isError(ret), "Must error");
|
CHECK(!ZSTD_isError(ret), "Must error");
|
||||||
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
|
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
|
||||||
}
|
}
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer with continue and flush : ", testNb++);
|
/* stableSrc + streaming */
|
||||||
in.src = CNBuffer;
|
DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream, flushStream and endStream : ", testNb++);
|
||||||
in.size = CNBufferSize;
|
CHECK_Z( ZSTD_initCStream(cctx, 1) );
|
||||||
in.pos = 0;
|
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
|
||||||
out.pos = 0;
|
{ ZSTD_inBuffer inBuf;
|
||||||
out.size = compressedBufferSize;
|
ZSTD_outBuffer outBuf;
|
||||||
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only));
|
const size_t nonZeroStartPos = 18;
|
||||||
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
|
const size_t inputSize = 500;
|
||||||
CHECK(!ZSTD_isError(ret), "Must error");
|
inBuf.src = CNBuffer;
|
||||||
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
|
inBuf.size = 100;
|
||||||
}
|
inBuf.pos = nonZeroStartPos;
|
||||||
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only));
|
outBuf.dst = (char*)(compressedBuffer)+cSize;
|
||||||
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
|
outBuf.size = ZSTD_compressBound(inputSize);
|
||||||
CHECK(!ZSTD_isError(ret), "Must error");
|
outBuf.pos = 0;
|
||||||
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
|
CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
|
||||||
}
|
inBuf.size = 200;
|
||||||
|
CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
|
||||||
|
CHECK_Z( ZSTD_flushStream(cctx, &outBuf) );
|
||||||
|
inBuf.size = nonZeroStartPos + inputSize;
|
||||||
|
CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
|
||||||
|
CHECK(ZSTD_endStream(cctx, &outBuf) != 0, "compression should be successful and fully flushed");
|
||||||
|
{ const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
|
||||||
|
void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
|
||||||
|
const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
|
||||||
|
CHECK_Z(decSize);
|
||||||
|
CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
|
||||||
|
CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
|
||||||
|
} }
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer allocated size : ", testNb++);
|
/* stableSrc + streaming */
|
||||||
|
DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream2, using different end directives : ", testNb++);
|
||||||
|
CHECK_Z( ZSTD_initCStream(cctx, 1) );
|
||||||
|
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
|
||||||
|
{ ZSTD_inBuffer inBuf;
|
||||||
|
ZSTD_outBuffer outBuf;
|
||||||
|
const size_t nonZeroStartPos = 18;
|
||||||
|
const size_t inputSize = 500;
|
||||||
|
inBuf.src = CNBuffer;
|
||||||
|
inBuf.size = 100;
|
||||||
|
inBuf.pos = nonZeroStartPos;
|
||||||
|
outBuf.dst = (char*)(compressedBuffer)+cSize;
|
||||||
|
outBuf.size = ZSTD_compressBound(inputSize);
|
||||||
|
outBuf.pos = 0;
|
||||||
|
CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
|
||||||
|
inBuf.size = 200;
|
||||||
|
CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
|
||||||
|
CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_flush) );
|
||||||
|
inBuf.size = nonZeroStartPos + inputSize;
|
||||||
|
CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
|
||||||
|
CHECK( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_end) != 0, "compression should be successful and fully flushed");
|
||||||
|
{ const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
|
||||||
|
void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
|
||||||
|
const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
|
||||||
|
CHECK_Z(decSize);
|
||||||
|
CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
|
||||||
|
CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
|
||||||
|
} }
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer: context size : ", testNb++);
|
||||||
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
|
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
|
||||||
|
DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
|
||||||
CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
|
CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
|
||||||
CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
|
CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
|
||||||
cctxSize1 = cctxSize;
|
cctxSize1 = cctxSize;
|
||||||
@ -900,8 +945,10 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
|
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
|
||||||
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
|
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
|
||||||
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
|
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
|
||||||
|
in.src = CNBuffer;
|
||||||
in.pos = out.pos = 0;
|
in.pos = out.pos = 0;
|
||||||
in.size = MIN(CNBufferSize, 10);
|
in.size = MIN(CNBufferSize, 10);
|
||||||
|
out.size = compressedBufferSize;
|
||||||
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
|
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
|
||||||
in.pos = 0;
|
in.pos = 0;
|
||||||
in.size = CNBufferSize - in.size;
|
in.size = CNBufferSize - in.size;
|
||||||
@ -916,12 +963,13 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
in.pos = out.pos = 0;
|
in.pos = out.pos = 0;
|
||||||
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
|
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
|
||||||
CHECK(!ZSTD_isError(ret), "Must have errored");
|
CHECK(!ZSTD_isError(ret), "Must have errored");
|
||||||
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_dstBuffer_wrong), "Must be this error");
|
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
|
||||||
}
|
}
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer allocated size : ", testNb++);
|
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer: context size : ", testNb++);
|
||||||
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
|
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
|
||||||
|
DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
|
||||||
CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
|
CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
|
||||||
CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
|
CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user