1
0
mirror of https://github.com/facebook/zstd.git synced 2025-07-29 11:21:22 +03:00

contentSizeFlag enabled by default would also fail for streaming and MT operations

fixed
This commit is contained in:
Yann Collet
2017-10-13 18:32:06 -07:00
parent fb44516641
commit 3c1e3f8ec9
3 changed files with 28 additions and 25 deletions

View File

@ -2053,6 +2053,8 @@ size_t ZSTD_compressBegin_advanced_internal(
ZSTD_CCtx_params params, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize) unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal");
DEBUGLOG(4, "contentSizeFlag : %u", params.fParams.contentSizeFlag);
/* compression parameters verification and optimization */ /* compression parameters verification and optimization */
CHECK_F( ZSTD_checkCParams(params.cParams) ); CHECK_F( ZSTD_checkCParams(params.cParams) );
return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL, return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL,
@ -2281,7 +2283,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
ZSTD_dictMode_e dictMode, ZSTD_dictMode_e dictMode,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem) ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
{ {
DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); DEBUGLOG(4, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
@ -2356,9 +2358,9 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
+ cctxSize; + cctxSize;
ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
void* ptr; void* ptr;
DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7); DEBUGLOG(4, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
if ((size_t)workspace & 7) return NULL; /* 8-aligned */ if ((size_t)workspace & 7) return NULL; /* 8-aligned */
DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u", DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u",
(U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize)); (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
if (workspaceSize < neededSize) return NULL; if (workspaceSize < neededSize) return NULL;
@ -2394,7 +2396,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
{ ZSTD_CCtx_params params = cctx->requestedParams; { ZSTD_CCtx_params params = cctx->requestedParams;
params.cParams = ZSTD_getCParamsFromCDict(cdict); params.cParams = ZSTD_getCParamsFromCDict(cdict);
params.fParams = fParams; params.fParams = fParams;
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced"); DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
return ZSTD_compressBegin_internal(cctx, return ZSTD_compressBegin_internal(cctx,
NULL, 0, ZSTD_dm_auto, NULL, 0, ZSTD_dm_auto,
cdict, cdict,
@ -2409,7 +2411,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{ {
ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0); return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
} }
@ -2585,7 +2587,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
ZSTD_CCtx_params const cctxParams = ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params);
return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0); return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
} }
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)

View File

@ -327,6 +327,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool); ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool);
const void* const src = (const char*)job->srcStart + job->dictSize; const void* const src = (const char*)job->srcStart + job->dictSize;
buffer_t dstBuff = job->dstBuff; buffer_t dstBuff = job->dstBuff;
DEBUGLOG(5, "ZSTDMT_compressChunk")
DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u",
job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize); job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
@ -344,22 +345,22 @@ void ZSTDMT_compressChunk(void* jobDescription)
job->dstBuff = dstBuff; job->dstBuff = dstBuff;
} }
if (job->cdict) { /* should only happen for first segment */ if (job->cdict) {
size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize); size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize);
DEBUGLOG(5, "using CDict"); DEBUGLOG(4, "ZSTDMT_compressChunk, using CDict");
assert(job->firstChunk); /* should only happen for first segment */
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
} else { /* srcStart points at reloaded section */ } else { /* srcStart points at reloaded section */
if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */ ZSTD_CCtx_params jobParams = job->params;
{ ZSTD_CCtx_params jobParams = job->params; size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
size_t const forceWindowError = U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN;
ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk); /* load dictionary in "content-only" mode (no header analysis) */
/* Force loading dictionary in "content-only" mode (no header analysis) */ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, pledgedSrcSize);
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize); if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) { job->cSize = initError;
job->cSize = initError; goto _endJob;
goto _endJob; }
} }
} }
if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */ if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */
size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0); size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);
if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; } if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
@ -367,7 +368,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
} }
DEBUGLOG(5, "Compressing : "); DEBUGLOG(5, "Compressing : ");
DEBUG_PRINTHEX(4, job->srcStart, 12); DEBUG_PRINTHEX(6, job->srcStart, 12);
job->cSize = (job->lastChunk) ? job->cSize = (job->lastChunk) ?
ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) : ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize); ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
@ -610,6 +611,7 @@ static size_t ZSTDMT_compress_advanced_internal(
assert(jobParams.nbThreads == 0); assert(jobParams.nbThreads == 0);
assert(mtctx->cctxPool->totalCCtx == params.nbThreads); assert(mtctx->cctxPool->totalCCtx == params.nbThreads);
DEBUGLOG(4, "ZSTDMT_compress_advanced_internal");
DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize); DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize);
if (nbChunks==1) { /* fallback to single-thread mode */ if (nbChunks==1) { /* fallback to single-thread mode */
ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0];
@ -641,7 +643,7 @@ static size_t ZSTDMT_compress_advanced_internal(
mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize; mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize;
mtctx->jobs[u].dictSize = dictSize; mtctx->jobs[u].dictSize = dictSize;
mtctx->jobs[u].srcSize = chunkSize; mtctx->jobs[u].srcSize = chunkSize;
mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL; mtctx->jobs[u].cdict = (u==0) ? cdict : NULL;
mtctx->jobs[u].fullFrameSize = srcSize; mtctx->jobs[u].fullFrameSize = srcSize;
mtctx->jobs[u].params = jobParams; mtctx->jobs[u].params = jobParams;
/* do not calculate checksum within sections, but write it in header for first section */ /* do not calculate checksum within sections, but write it in header for first section */
@ -777,7 +779,6 @@ size_t ZSTDMT_initCStream_internal(
zcs->params = params; zcs->params = params;
zcs->frameContentSize = pledgedSrcSize; zcs->frameContentSize = pledgedSrcSize;
if (dict) { if (dict) {
DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */ ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */
@ -785,7 +786,6 @@ size_t ZSTDMT_initCStream_internal(
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
} else { } else {
DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal);
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = NULL; zcs->cdictLocal = NULL;
zcs->cdict = cdict; zcs->cdict = cdict;

View File

@ -190,9 +190,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic compression test */ /* Basic compression test */
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) ); CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
outBuff.dst = (char*)(compressedBuffer)+cSize; outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = compressedBufferSize; assert(compressedBufferSize > cSize);
outBuff.size = compressedBufferSize - cSize;
outBuff.pos = 0; outBuff.pos = 0;
inBuff.src = CNBuffer; inBuff.src = CNBuffer;
inBuff.size = CNBufferSize; inBuff.size = CNBufferSize;