mirror of
https://github.com/facebook/zstd.git
synced 2025-07-29 11:21:22 +03:00
contentSizeFlag enabled by default would also fail for streaming and MT operations
fixed
This commit is contained in:
@ -2053,6 +2053,8 @@ size_t ZSTD_compressBegin_advanced_internal(
|
|||||||
ZSTD_CCtx_params params,
|
ZSTD_CCtx_params params,
|
||||||
unsigned long long pledgedSrcSize)
|
unsigned long long pledgedSrcSize)
|
||||||
{
|
{
|
||||||
|
DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal");
|
||||||
|
DEBUGLOG(4, "contentSizeFlag : %u", params.fParams.contentSizeFlag);
|
||||||
/* compression parameters verification and optimization */
|
/* compression parameters verification and optimization */
|
||||||
CHECK_F( ZSTD_checkCParams(params.cParams) );
|
CHECK_F( ZSTD_checkCParams(params.cParams) );
|
||||||
return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL,
|
return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL,
|
||||||
@ -2281,7 +2283,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
|
|||||||
ZSTD_dictMode_e dictMode,
|
ZSTD_dictMode_e dictMode,
|
||||||
ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
|
ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
|
||||||
{
|
{
|
||||||
DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
|
DEBUGLOG(4, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
|
||||||
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
|
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
|
||||||
|
|
||||||
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
|
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
|
||||||
@ -2356,9 +2358,9 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
|
|||||||
+ cctxSize;
|
+ cctxSize;
|
||||||
ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
|
ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
|
||||||
void* ptr;
|
void* ptr;
|
||||||
DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
|
DEBUGLOG(4, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
|
||||||
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
|
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
|
||||||
DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u",
|
DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u",
|
||||||
(U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
|
(U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
|
||||||
if (workspaceSize < neededSize) return NULL;
|
if (workspaceSize < neededSize) return NULL;
|
||||||
|
|
||||||
@ -2394,7 +2396,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
|
|||||||
{ ZSTD_CCtx_params params = cctx->requestedParams;
|
{ ZSTD_CCtx_params params = cctx->requestedParams;
|
||||||
params.cParams = ZSTD_getCParamsFromCDict(cdict);
|
params.cParams = ZSTD_getCParamsFromCDict(cdict);
|
||||||
params.fParams = fParams;
|
params.fParams = fParams;
|
||||||
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
|
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
|
||||||
return ZSTD_compressBegin_internal(cctx,
|
return ZSTD_compressBegin_internal(cctx,
|
||||||
NULL, 0, ZSTD_dm_auto,
|
NULL, 0, ZSTD_dm_auto,
|
||||||
cdict,
|
cdict,
|
||||||
@ -2409,7 +2411,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
|
|||||||
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
|
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
|
||||||
{
|
{
|
||||||
ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
|
ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
|
||||||
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
|
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
|
||||||
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
|
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2585,7 +2587,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
|
|||||||
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
|
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
|
||||||
ZSTD_CCtx_params const cctxParams =
|
ZSTD_CCtx_params const cctxParams =
|
||||||
ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
|
ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
|
||||||
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0);
|
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
|
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
|
||||||
|
@ -327,6 +327,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
|
|||||||
ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
|
ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
|
||||||
const void* const src = (const char*)job->srcStart + job->dictSize;
|
const void* const src = (const char*)job->srcStart + job->dictSize;
|
||||||
buffer_t dstBuff = job->dstBuff;
|
buffer_t dstBuff = job->dstBuff;
|
||||||
|
DEBUGLOG(5, "ZSTDMT_compressChunk")
|
||||||
DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
|
DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
|
||||||
job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
|
job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
|
||||||
|
|
||||||
@ -344,22 +345,22 @@ void ZSTDMT_compressChunk(void* jobDescription)
|
|||||||
job->dstBuff = dstBuff;
|
job->dstBuff = dstBuff;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (job->cdict) { /* should only happen for first segment */
|
if (job->cdict) {
|
||||||
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
|
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
|
||||||
DEBUGLOG(5, "using CDict");
|
DEBUGLOG(4, "ZSTDMT_compressChunk, using CDict");
|
||||||
|
assert(job->firstChunk); /* should only happen for first segment */
|
||||||
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
|
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
|
||||||
} else { /* srcStart points at reloaded section */
|
} else { /* srcStart points at reloaded section */
|
||||||
if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */
|
ZSTD_CCtx_params jobParams = job->params;
|
||||||
{ ZSTD_CCtx_params jobParams = job->params;
|
size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
|
||||||
size_t const forceWindowError =
|
U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN;
|
||||||
ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
|
/* load dictionary in "content-only" mode (no header analysis) */
|
||||||
/* Force loading dictionary in "content-only" mode (no header analysis) */
|
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, pledgedSrcSize);
|
||||||
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize);
|
if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
|
||||||
if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
|
job->cSize = initError;
|
||||||
job->cSize = initError;
|
goto _endJob;
|
||||||
goto _endJob;
|
}
|
||||||
}
|
}
|
||||||
} }
|
|
||||||
if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */
|
if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */
|
||||||
size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
|
size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
|
||||||
if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
|
if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
|
||||||
@ -367,7 +368,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
|
|||||||
}
|
}
|
||||||
|
|
||||||
DEBUGLOG(5, "Compressing : ");
|
DEBUGLOG(5, "Compressing : ");
|
||||||
DEBUG_PRINTHEX(4, job->srcStart, 12);
|
DEBUG_PRINTHEX(6, job->srcStart, 12);
|
||||||
job->cSize = (job->lastChunk) ?
|
job->cSize = (job->lastChunk) ?
|
||||||
ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
|
ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
|
||||||
ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
|
ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
|
||||||
@ -610,6 +611,7 @@ static size_t ZSTDMT_compress_advanced_internal(
|
|||||||
assert(jobParams.nbThreads == 0);
|
assert(jobParams.nbThreads == 0);
|
||||||
assert(mtctx->cctxPool->totalCCtx == params.nbThreads);
|
assert(mtctx->cctxPool->totalCCtx == params.nbThreads);
|
||||||
|
|
||||||
|
DEBUGLOG(4, "ZSTDMT_compress_advanced_internal");
|
||||||
DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
|
DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
|
||||||
if (nbChunks==1) { /* fallback to single-thread mode */
|
if (nbChunks==1) { /* fallback to single-thread mode */
|
||||||
ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
|
ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
|
||||||
@ -641,7 +643,7 @@ static size_t ZSTDMT_compress_advanced_internal(
|
|||||||
mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
|
mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
|
||||||
mtctx->jobs[u].dictSize = dictSize;
|
mtctx->jobs[u].dictSize = dictSize;
|
||||||
mtctx->jobs[u].srcSize = chunkSize;
|
mtctx->jobs[u].srcSize = chunkSize;
|
||||||
mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL;
|
mtctx->jobs[u].cdict = (u==0) ? cdict : NULL;
|
||||||
mtctx->jobs[u].fullFrameSize = srcSize;
|
mtctx->jobs[u].fullFrameSize = srcSize;
|
||||||
mtctx->jobs[u].params = jobParams;
|
mtctx->jobs[u].params = jobParams;
|
||||||
/* do not calculate checksum within sections, but write it in header for first section */
|
/* do not calculate checksum within sections, but write it in header for first section */
|
||||||
@ -777,7 +779,6 @@ size_t ZSTDMT_initCStream_internal(
|
|||||||
zcs->params = params;
|
zcs->params = params;
|
||||||
zcs->frameContentSize = pledgedSrcSize;
|
zcs->frameContentSize = pledgedSrcSize;
|
||||||
if (dict) {
|
if (dict) {
|
||||||
DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
|
|
||||||
ZSTD_freeCDict(zcs->cdictLocal);
|
ZSTD_freeCDict(zcs->cdictLocal);
|
||||||
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
|
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
|
||||||
ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */
|
ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */
|
||||||
@ -785,7 +786,6 @@ size_t ZSTDMT_initCStream_internal(
|
|||||||
zcs->cdict = zcs->cdictLocal;
|
zcs->cdict = zcs->cdictLocal;
|
||||||
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
|
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
|
||||||
} else {
|
} else {
|
||||||
DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
|
|
||||||
ZSTD_freeCDict(zcs->cdictLocal);
|
ZSTD_freeCDict(zcs->cdictLocal);
|
||||||
zcs->cdictLocal = NULL;
|
zcs->cdictLocal = NULL;
|
||||||
zcs->cdict = cdict;
|
zcs->cdict = cdict;
|
||||||
|
@ -190,9 +190,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
|
|||||||
|
|
||||||
/* Basic compression test */
|
/* Basic compression test */
|
||||||
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||||
CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) );
|
CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
|
||||||
outBuff.dst = (char*)(compressedBuffer)+cSize;
|
outBuff.dst = (char*)(compressedBuffer)+cSize;
|
||||||
outBuff.size = compressedBufferSize;
|
assert(compressedBufferSize > cSize);
|
||||||
|
outBuff.size = compressedBufferSize - cSize;
|
||||||
outBuff.pos = 0;
|
outBuff.pos = 0;
|
||||||
inBuff.src = CNBuffer;
|
inBuff.src = CNBuffer;
|
||||||
inBuff.size = CNBufferSize;
|
inBuff.size = CNBufferSize;
|
||||||
|
Reference in New Issue
Block a user