From 5d9b894e46c25eb4b861dbb251588a27f6d87e86 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 27 Jan 2017 13:30:18 -0800 Subject: [PATCH 01/10] Fixed status display for zstdmt There is a large buffering effect when using zstdmt in MT mode. Consequently, data is read first, pushed to workers, and only later will the compressed result come out. That means there is no longer immediate correlation between amount of data read, and amount of data written. This patch disables the displaying of % compression when multi-threading is enabled. It adds the displaying of total size when it can be determined (it usually can be determined for files, but not for stdin) so the user has a sense of "how far from the end" the compression compressed is. There is no modification to decompression side, since decompression is only single-threaded for now. --- programs/fileio.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 86da00131..353fbd54e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -78,14 +78,14 @@ * Macros ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ +#define DISPLAYUPDATE(l, ...) { if (g_displayLevel>=l) { \ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stdout); } } + if (g_displayLevel>=4) fflush(stdout); } } } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; @@ -373,7 +373,13 @@ static int FIO_compressFilename_internal(cRess_t ress, if (sizeCheck!=outBuff.pos) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); compressedfilesize += outBuff.pos; } } } - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100); +#ifdef ZSTD_MULTITHREAD + if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB", (U32)(readsize>>20)) + else DISPLAYUPDATE(2, "\rRead : %u / %u MB", (U32)(readsize>>20), (U32)(fileSize>>20)); +#else + if (!fileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(readsize>>20), (double)compressedfilesize/readsize*100) + else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(readsize>>20), (U32)(fileSize>>20), (double)compressedfilesize/readsize*100); +#endif } /* End of Frame */ From f6d4a786fc188dfd8498d437879f5f0bd5bc4182 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 27 Jan 2017 15:55:30 -0800 Subject: [PATCH 02/10] reduced zstdmt latency when using small custom section sizes with high compression levels Previous version was requiring a fairly large initial amount of input data before starting to create compression jobs. This new version starts the process much sooner. --- lib/compress/zstdmt_compress.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 5f0bf2ab1..ca9bf6a26 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -273,6 +273,7 @@ struct ZSTDMT_CCtx_s { pthread_mutex_t jobCompleted_mutex; pthread_cond_t jobCompleted_cond; size_t targetSectionSize; + size_t marginSize; size_t inBuffSize; size_t dictSize; size_t targetDictSize; @@ -514,8 +515,9 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, zcs->frameContentSize = pledgedSrcSize; zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2); zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize); + zcs->marginSize = zcs->targetSectionSize >> 2; zcs->targetDictSize = zcs->overlapWrLog < 10 ? (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapWrLog) : 0; - zcs->inBuffSize = zcs->targetSectionSize + ((size_t)1 << zcs->params.cParams.windowLog) /* margin */ + zcs->targetDictSize; + zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize; zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize); if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation); zcs->inBuff.filled = 0; @@ -680,6 +682,7 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { + size_t const newJobThreshold = zcs->dictSize + zcs->targetSectionSize + zcs->marginSize; if (zcs->frameEnded) return ERROR(stage_wrong); /* current frame being ended. Only flush is allowed. Restart with init */ if (zcs->nbThreads==1) return ZSTD_compressStream(zcs->cstream, output, input); @@ -690,7 +693,7 @@ size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBu zcs->inBuff.filled += toLoad; } - if ( (zcs->inBuff.filled == zcs->inBuffSize) /* filled enough : let's compress */ + if ( (zcs->inBuff.filled >= newJobThreshold) /* filled enough : let's compress */ && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) { /* avoid overwriting job round buffer */ CHECK_F( ZSTDMT_createCompressionJob(zcs, zcs->targetSectionSize, 0) ); } From 88df1aed6119d2e98ae683c3c348c1a0f5504bb3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 Jan 2017 11:00:00 -0800 Subject: [PATCH 03/10] changed advanced parameter overlapLog Follows a positive logic (increasing value => increasing overlap) which is easier to use --- lib/compress/zstdmt_compress.c | 10 +++++----- lib/compress/zstdmt_compress.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index ca9bf6a26..07c7c1b38 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -286,7 +286,7 @@ struct ZSTDMT_CCtx_s { unsigned nextJobID; unsigned frameEnded; unsigned allJobsCompleted; - unsigned overlapWrLog; + unsigned overlapRLog; unsigned long long frameContentSize; size_t sectionSize; ZSTD_CDict* cdict; @@ -309,7 +309,7 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(unsigned nbThreads) cctx->jobIDMask = nbJobs - 1; cctx->allJobsCompleted = 1; cctx->sectionSize = 0; - cctx->overlapWrLog = 3; + cctx->overlapRLog = 3; cctx->factory = POOL_create(nbThreads, 1); cctx->buffPool = ZSTDMT_createBufferPool(nbThreads); cctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads); @@ -369,8 +369,8 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, case ZSTDMT_p_sectionSize : mtctx->sectionSize = value; return 0; - case ZSTDMT_p_overlapSectionRLog : - mtctx->overlapWrLog = value; + case ZSTDMT_p_overlapSectionLog : + mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value; return 0; default : return ERROR(compressionParameter_unsupported); @@ -516,7 +516,7 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2); zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize); zcs->marginSize = zcs->targetSectionSize >> 2; - zcs->targetDictSize = zcs->overlapWrLog < 10 ? (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapWrLog) : 0; + zcs->targetDictSize = (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog); zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize; zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize); if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation); diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 92de52d65..acd03b37e 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -53,7 +53,7 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* d * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ typedef enum { ZSTDMT_p_sectionSize, /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */ - ZSTDMT_p_overlapSectionRLog /* reverse log of overlapped section; 0 == use a complete window, 3(default) == use 1/8th of window, values >=10 means no overlap */ + ZSTDMT_p_overlapSectionLog /* Log of overlapped section; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */ } ZSDTMT_parameter; /* ZSTDMT_setMTCtxParameter() : From 6be2337c2697f7437bb3fc802b6be4296206305f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 Jan 2017 11:17:26 -0800 Subject: [PATCH 04/10] added command --block-size= for Multi-threading only. alias : -B# --- programs/fileio.c | 14 +++++++++++--- programs/fileio.h | 1 + programs/zstdcli.c | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 353fbd54e..abaa15e53 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -124,6 +124,13 @@ void FIO_setBlockSize(unsigned blockSize) { #endif g_blockSize = blockSize; } +static const U32 g_overlapLogNotSet = 9999; +static U32 g_overlapLog = g_overlapLogNotSet; +void FIO_setOverlapLog(unsigned overlapLog){ + if (overlapLog && g_nbThreads==1) + DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); + g_overlapLog = overlapLog; +} /*-************************************* @@ -272,8 +279,10 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, #ifdef ZSTD_MULTITHREAD ress.cctx = ZSTDMT_createCCtx(g_nbThreads); if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream"); - if (cLevel==ZSTD_maxCLevel()) - ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionRLog, 0); /* use complete window for overlap */ + if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==g_overlapLogNotSet)) + ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 0); /* use complete window for overlap */ + if (g_overlapLog != g_overlapLogNotSet) + ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog); #else ress.cctx = ZSTD_createCStream(); if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream"); @@ -355,7 +364,6 @@ static int FIO_compressFilename_internal(cRess_t ress, size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); if (inSize==0) break; readsize += inSize; - DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(readsize>>20)); { ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; while (inBuff.pos != inBuff.size) { /* note : is there any possibility of endless loop ? for example, if outBuff is not large enough ? */ diff --git a/programs/fileio.h b/programs/fileio.h index 11178bcca..daff0312e 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -43,6 +43,7 @@ void FIO_setRemoveSrcFile(unsigned flag); void FIO_setMemLimit(unsigned memLimit); void FIO_setNbThreads(unsigned nbThreads); void FIO_setBlockSize(unsigned blockSize); +void FIO_setOverlapLog(unsigned overlapLog); /*-************************************* diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 64f2c919c..c6c8cd2bd 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -370,6 +370,7 @@ int main(int argCount, const char* argv[]) if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; } /* fall-through, will trigger bad_usage() later on */ } From cd23dd24af69b3f9b500502c1e10c3047e75ac4b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 Jan 2017 12:46:35 -0800 Subject: [PATCH 05/10] zstreamtest uses random overlapLog for fuzzing --- tests/zstreamtest.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index bef8734c7..c053172e9 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -834,9 +834,10 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp DISPLAYLEVEL(5, "Init with windowLog = %u \n", params.cParams.windowLog); params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; - { size_t const initError = ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize); - CHECK (ZSTD_isError(initError),"ZSTDMT_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); - } } } + { size_t const initError = ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize); + CHECK (ZSTD_isError(initError),"ZSTDMT_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); } + ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12); + } } /* multi-segments compression test */ XXH64_reset(&xxhState, 0); From 92c98a5b216e97fdb257e13e5cbdd06256d9f5d0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 Jan 2017 12:50:31 -0800 Subject: [PATCH 06/10] zstreamtest uses random section sizes for fuzzing --- tests/zstreamtest.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index c053172e9..aeaf02b3f 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -837,6 +837,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp { size_t const initError = ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize); CHECK (ZSTD_isError(initError),"ZSTDMT_initCStream_advanced error : %s", ZSTD_getErrorName(initError)); } ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapSectionLog, FUZ_rand(&lseed) % 12); + ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_sectionSize, FUZ_rand(&lseed) % (2*maxTestSize+1)); } } /* multi-segments compression test */ From 6ccd37c8d43cf3bad135c8bbbcd3a6513196b0d2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 Jan 2017 13:07:24 -0800 Subject: [PATCH 07/10] cli : added advanced parameter overlapLog as a hidden (undocumented) parameter for now --- programs/zstdcli.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c6c8cd2bd..c8cdafb56 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -186,7 +186,7 @@ static unsigned readU32FromChar(const char** stringPtr) } /** longCommandWArg() : - * check is *stringPtr is the same as longCommand. + * check if *stringPtr is the same as longCommand. * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. * @return 0 and doesn't modify *stringPtr otherwise. */ @@ -220,6 +220,10 @@ static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t *para return 1; } #endif + + +static const U32 g_overlapLogDefault = 9999; +static U32 g_overlapLog = g_overlapLogDefault; /** parseCompressionParameters() : * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params * @return 1 means that compression parameters were correct @@ -235,6 +239,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } return 0; } @@ -629,6 +634,7 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOCOMPRESS FIO_setNbThreads(nbThreads); FIO_setBlockSize((U32)blockSize); + if (g_overlapLog!=g_overlapLogDefault) FIO_setOverlapLog(g_overlapLog); if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams); else From 3672d06d067fda6d87d99c670d49e2b52a083c07 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 Jan 2017 13:35:45 -0800 Subject: [PATCH 08/10] zstdmt : section size is set to be a minimum of overlapSize the minimum size condition size is applied transparently (no warning, no error) like previous minimum section size condition (1 KB) which still applies. --- lib/compress/zstdmt_compress.c | 7 ++++++- programs/fileio.c | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 07c7c1b38..d122d8295 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -370,6 +370,7 @@ size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSDTMT_parameter parameter, mtctx->sectionSize = value; return 0; case ZSTDMT_p_overlapSectionLog : + DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value); mtctx->overlapRLog = (value >= 9) ? 0 : 9 - value; return 0; default : @@ -513,10 +514,14 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, if (zcs->cdict == NULL) return ERROR(memory_allocation); } } zcs->frameContentSize = pledgedSrcSize; + zcs->targetDictSize = (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog); + DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog); + DEBUGLOG(3, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10)); zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2); zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize); + zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize); + DEBUGLOG(3, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10)); zcs->marginSize = zcs->targetSectionSize >> 2; - zcs->targetDictSize = (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog); zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize + zcs->marginSize; zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->buffPool, zcs->inBuffSize); if (zcs->inBuff.buffer.start == NULL) return ERROR(memory_allocation); diff --git a/programs/fileio.c b/programs/fileio.c index abaa15e53..568b4f115 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -280,7 +280,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, ress.cctx = ZSTDMT_createCCtx(g_nbThreads); if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream"); if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==g_overlapLogNotSet)) - ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 0); /* use complete window for overlap */ + ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9); /* use complete window for overlap */ if (g_overlapLog != g_overlapLogNotSet) ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog); #else From 8d8513fb64ca936b26da4153217ee125930d6958 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 Jan 2017 14:37:08 -0800 Subject: [PATCH 09/10] fixed C constant restrictions --- programs/fileio.c | 8 ++++---- programs/zstdcli.c | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 568b4f115..960c6e3d9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -124,8 +124,8 @@ void FIO_setBlockSize(unsigned blockSize) { #endif g_blockSize = blockSize; } -static const U32 g_overlapLogNotSet = 9999; -static U32 g_overlapLog = g_overlapLogNotSet; +#define FIO_OVERLAP_LOG_NOTSET 9999 +static U32 g_overlapLog = FIO_OVERLAP_LOG_NOTSET; void FIO_setOverlapLog(unsigned overlapLog){ if (overlapLog && g_nbThreads==1) DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); @@ -279,9 +279,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, #ifdef ZSTD_MULTITHREAD ress.cctx = ZSTDMT_createCCtx(g_nbThreads); if (ress.cctx == NULL) EXM_THROW(30, "zstd: allocation error : can't create ZSTD_CStream"); - if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==g_overlapLogNotSet)) + if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET)) ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9); /* use complete window for overlap */ - if (g_overlapLog != g_overlapLogNotSet) + if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog); #else ress.cctx = ZSTD_createCStream(); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c8cdafb56..c59bf0c3a 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -63,6 +63,8 @@ static const char* g_defaultDictName = "dictionary"; static const unsigned g_defaultMaxDictSize = 110 KB; static const int g_defaultDictCLevel = 3; static const unsigned g_defaultSelectivityLevel = 9; +#define OVERLAP_LOG_DEFAULT 9999 +static U32 g_overlapLog = OVERLAP_LOG_DEFAULT; /*-************************************ @@ -222,8 +224,6 @@ static unsigned parseCoverParameters(const char* stringPtr, COVER_params_t *para #endif -static const U32 g_overlapLogDefault = 9999; -static U32 g_overlapLog = g_overlapLogDefault; /** parseCompressionParameters() : * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params * @return 1 means that compression parameters were correct @@ -634,7 +634,7 @@ int main(int argCount, const char* argv[]) #ifndef ZSTD_NOCOMPRESS FIO_setNbThreads(nbThreads); FIO_setBlockSize((U32)blockSize); - if (g_overlapLog!=g_overlapLogDefault) FIO_setOverlapLog(g_overlapLog); + if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog); if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams); else From b2e1b3d670f7e73f3b73b02c3734002feb24235f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 30 Jan 2017 14:54:46 -0800 Subject: [PATCH 10/10] fixed overlapLog==0 => no overlap --- lib/compress/zstdmt_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index d122d8295..6e63e8801 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -514,7 +514,7 @@ static size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, if (zcs->cdict == NULL) return ERROR(memory_allocation); } } zcs->frameContentSize = pledgedSrcSize; - zcs->targetDictSize = (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog); + zcs->targetDictSize = (zcs->overlapRLog>=9) ? 0 : (size_t)1 << (zcs->params.cParams.windowLog - zcs->overlapRLog); DEBUGLOG(4, "overlapRLog : %u ", zcs->overlapRLog); DEBUGLOG(3, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10)); zcs->targetSectionSize = zcs->sectionSize ? zcs->sectionSize : (size_t)1 << (zcs->params.cParams.windowLog + 2);