diff --git a/programs/Makefile b/programs/Makefile index 1b21281b4..e1c5f50b6 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -37,8 +37,7 @@ endif CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder \ - -DXXH_NAMESPACE=ZSTD_ \ - -DZSTD_NEWAPI + -DXXH_NAMESPACE=ZSTD_ CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ diff --git a/programs/bench.c b/programs/bench.c index efec1892a..5b67d5c76 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -287,7 +287,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U32 nbLoops = 0; ZSTD_CDict* cdict = NULL; -#ifdef ZSTD_NEWAPI ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag); @@ -306,25 +305,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); -#else - size_t const avgSize = MIN(blockSize, (srcSize / nbFiles)); - ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); - ZSTD_customMem const cmem = { NULL, NULL, NULL }; - if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog; - if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog; - if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog; - if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog; - if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength; - if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy; - cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem); - if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); -#endif do { U32 blockNb; for (blockNb=0; blockNbwindowLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) ); - /* multi-threading */ - DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); - /* dictionary */ - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */ - CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size, when available (note: supposed to be default anyway) */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); + /* compression level */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) ); + /* long distance matching */ + CHECK( ZSTD_CCtx_setParameter( + ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) ); + if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) ); } -#elif defined(ZSTD_MULTITHREAD) - { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); - params.fParams.checksumFlag = g_checksumFlag; - params.fParams.noDictIDFlag = !g_dictIDFlag; - if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; - if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog; - if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog; - if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; - if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; - if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy; - CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) ); - ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize); + if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) { + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) ); } -#else - { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); - params.fParams.checksumFlag = g_checksumFlag; - params.fParams.noDictIDFlag = !g_dictIDFlag; - if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; - if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog; - if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog; - if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; - if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; - if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy; - CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) ); - } -#endif + /* compression parameters */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) ); + /* multi-threading */ + DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); + /* dictionary */ + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */ + CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ + free(dictBuffer); } @@ -509,11 +460,7 @@ static void FIO_freeCResources(cRess_t ress) { free(ress.srcBuffer); free(ress.dstBuffer); -#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD) - ZSTDMT_freeCCtx(ress.cctx); -#else ZSTD_freeCStream(ress.cctx); /* never fails */ -#endif } @@ -814,14 +761,8 @@ static int FIO_compressFilename_internal(cRess_t ress, } /* init */ -#ifdef ZSTD_NEWAPI - if (fileSize!=UTIL_FILESIZE_UNKNOWN) /* when src is stdin, fileSize==0, but is effectively unknown */ + if (fileSize != UTIL_FILESIZE_UNKNOWN) ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); -#elif defined(ZSTD_MULTITHREAD) - CHECK( ZSTDMT_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) ); -#else - CHECK( ZSTD_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) ); -#endif /* Main compression loop */ while (1) { @@ -833,14 +774,8 @@ static int FIO_compressFilename_internal(cRess_t ress, while (inBuff.pos != inBuff.size) { ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; -#ifdef ZSTD_NEWAPI CHECK( ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, ZSTD_e_continue) ); -#elif defined(ZSTD_MULTITHREAD) - CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) ); -#else - CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) ); -#endif /* Write compressed stream */ if (outBuff.pos) { @@ -871,21 +806,15 @@ static int FIO_compressFilename_internal(cRess_t ress, { size_t result = 1; while (result != 0) { ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; -#ifdef ZSTD_NEWAPI ZSTD_inBuffer inBuff = { NULL, 0, 0 }; result = ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, ZSTD_e_end); -#elif defined(ZSTD_MULTITHREAD) - result = ZSTDMT_endStream(ress.cctx, &outBuff); -#else - result = ZSTD_endStream(ress.cctx, &outBuff); -#endif if (ZSTD_isError(result)) { EXM_THROW(26, "Compression error during frame end : %s", ZSTD_getErrorName(result)); } { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); - if (sizeCheck!=outBuff.pos) + if (sizeCheck != outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); } compressedfilesize += outBuff.pos;