diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 146219d29..e03a457b1 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -61,7 +61,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 7 -#define ZSTD_VERSION_RELEASE 3 +#define ZSTD_VERSION_RELEASE 4 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str @@ -262,6 +262,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v /*-************************************* * Advanced compression functions ***************************************/ +/*! ZSTD_estimateCCtxSize() : + * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. + * `frameContentSize` is an optional parameter, provide `0` if unknown */ +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize); + /*! ZSTD_createCCtx_advanced() : * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index cd55c7ae7..70e0a7409 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -175,6 +175,11 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) return 0; /* reserved as a potential error code in the future */ } +size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx) +{ + return sizeof(*cctx) + cctx->workSpaceSize; +} + const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */ { return &(ctx->seqStore); @@ -244,17 +249,27 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u } -size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface, for paramagrill */ +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned long long frameContentSize) { - ZSTD_CCtx* const zc = ZSTD_createCCtx(); - ZSTD_parameters params; - memset(¶ms, 0, sizeof(params)); - params.cParams = cParams; - params.fParams.contentSizeFlag = 1; - ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0); - { size_t const ccsize = sizeof(*zc) + zc->workSpaceSize; - ZSTD_freeCCtx(zc); - return ccsize; } + const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + const U32 divider = (cParams.searchLength==3) ? 3 : 4; + const size_t maxNbSeq = blockSize / divider; + const size_t tokenSpace = blockSize + 11*maxNbSeq; + + const size_t chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); + const size_t hSize = ((size_t)1) << cParams.hashLog; + const U32 hashLog3 = (cParams.searchLength>3) ? 0 : + ( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX : + ((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) ); + const size_t h3Size = ((size_t)1) << hashLog3; + const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + + size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1< dstCapacity) return ERROR(dstSize_tooSmall); memset(dst, byte, length); @@ -1001,7 +1001,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_generateNxByte(op, oend-op, *ip, blockProperties.origSize); + decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); break; case bt_end : /* end of frame */ diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 6cf4ccd89..3078748f2 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -48,7 +48,7 @@ #endif #include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ #include "zstd.h" #include "datagen.h" #include "xxhash.h" @@ -274,6 +274,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, const int startTime =BMK_GetMilliStart(); DISPLAY("\r%79s\r", ""); + memset(¶ms, 0, sizeof(params)); params.cParams = cParams; params.fParams.contentSizeFlag = 0; for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { @@ -407,8 +408,6 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz BMK_printWinners2(stdout, winners, srcSize); } -size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters params); /* hidden interface, declared here */ - static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx) @@ -442,8 +441,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_sizeofCCtx(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_sizeofCCtx(winners[cLevel].params); + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params, srcSize); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params, srcSize); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); @@ -453,7 +452,6 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log((double)testResult.dSpeed); double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed); - if (W_DMemUsed_note < O_DMemUsed_note) { /* uses too much Decompression memory for too little benefit */ if (W_ratio > O_ratio)