From 3c1e3f8ec90a09558bda9b860fc1cbfdd322919c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 13 Oct 2017 18:32:06 -0700 Subject: [PATCH] contentSizeFlag enabled by default would also fail for streaming and MT operations fixed --- lib/compress/zstd_compress.c | 14 ++++++++------ lib/compress/zstdmt_compress.c | 34 +++++++++++++++++----------------- tests/zstreamtest.c | 5 +++-- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b42e6996f..49b532b94 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2053,6 +2053,8 @@ size_t ZSTD_compressBegin_advanced_internal( ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal"); + DEBUGLOG(4, "contentSizeFlag : %u", params.fParams.contentSizeFlag); /* compression parameters verification and optimization */ CHECK_F( ZSTD_checkCParams(params.cParams) ); return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL, @@ -2281,7 +2283,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_dictMode_e dictMode, ZSTD_compressionParameters cParams, ZSTD_customMem customMem) { - DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); + DEBUGLOG(4, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); @@ -2356,9 +2358,9 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize, + cctxSize; ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; void* ptr; - DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7); + DEBUGLOG(4, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7); if ((size_t)workspace & 7) return NULL; /* 8-aligned */ - DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u", + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; @@ -2394,7 +2396,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( { ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ZSTD_getCParamsFromCDict(cdict); params.fParams = fParams; - DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced"); + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); return ZSTD_compressBegin_internal(cctx, NULL, 0, ZSTD_dm_auto, cdict, @@ -2409,7 +2411,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0); } @@ -2585,7 +2587,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0); + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN); } size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 7831cd3bd..17c69c9df 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -327,6 +327,7 @@ void ZSTDMT_compressChunk(void* jobDescription) ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool); const void* const src = (const char*)job->srcStart + job->dictSize; buffer_t dstBuff = job->dstBuff; + DEBUGLOG(5, "ZSTDMT_compressChunk") DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize); @@ -344,22 +345,22 @@ void ZSTDMT_compressChunk(void* jobDescription) job->dstBuff = dstBuff; } - if (job->cdict) { /* should only happen for first segment */ + if (job->cdict) { size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize); - DEBUGLOG(5, "using CDict"); + DEBUGLOG(4, "ZSTDMT_compressChunk, using CDict"); + assert(job->firstChunk); /* should only happen for first segment */ if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } } else { /* srcStart points at reloaded section */ - if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */ - { ZSTD_CCtx_params jobParams = job->params; - size_t const forceWindowError = - ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk); - /* Force loading dictionary in "content-only" mode (no header analysis) */ - size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize); - if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) { - job->cSize = initError; - goto _endJob; - } - } } + ZSTD_CCtx_params jobParams = job->params; + size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk); + U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN; + /* load dictionary in "content-only" mode (no header analysis) */ + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, pledgedSrcSize); + if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) { + job->cSize = initError; + goto _endJob; + } + } if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */ size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0); if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; } @@ -367,7 +368,7 @@ void ZSTDMT_compressChunk(void* jobDescription) } DEBUGLOG(5, "Compressing : "); - DEBUG_PRINTHEX(4, job->srcStart, 12); + DEBUG_PRINTHEX(6, job->srcStart, 12); job->cSize = (job->lastChunk) ? ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) : ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize); @@ -610,6 +611,7 @@ static size_t ZSTDMT_compress_advanced_internal( assert(jobParams.nbThreads == 0); assert(mtctx->cctxPool->totalCCtx == params.nbThreads); + DEBUGLOG(4, "ZSTDMT_compress_advanced_internal"); DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize); if (nbChunks==1) { /* fallback to single-thread mode */ ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; @@ -641,7 +643,7 @@ static size_t ZSTDMT_compress_advanced_internal( mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize; mtctx->jobs[u].dictSize = dictSize; mtctx->jobs[u].srcSize = chunkSize; - mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL; + mtctx->jobs[u].cdict = (u==0) ? cdict : NULL; mtctx->jobs[u].fullFrameSize = srcSize; mtctx->jobs[u].params = jobParams; /* do not calculate checksum within sections, but write it in header for first section */ @@ -777,7 +779,6 @@ size_t ZSTDMT_initCStream_internal( zcs->params = params; zcs->frameContentSize = pledgedSrcSize; if (dict) { - DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal); zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */ @@ -785,7 +786,6 @@ size_t ZSTDMT_initCStream_internal( zcs->cdict = zcs->cdictLocal; if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); } else { - DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal); zcs->cdictLocal = NULL; zcs->cdict = cdict; diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 8b4c83694..36eede293 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -190,9 +190,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Basic compression test */ DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1) ); + CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) ); outBuff.dst = (char*)(compressedBuffer)+cSize; - outBuff.size = compressedBufferSize; + assert(compressedBufferSize > cSize); + outBuff.size = compressedBufferSize - cSize; outBuff.pos = 0; inBuff.src = CNBuffer; inBuff.size = CNBufferSize;