lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-05 19:15:58 +03:00
Code Activity

IO and bench : ZSTD_NEWAPI is the only remaining code path

Browse Source 
removed the other 2 code paths (single thread, and ZSTDMT ones)
keeping only the new advanced API, for easier code coverage.

It shall also fix identified issue with Visual Studio
which doesn't have ZSTD_NEWAPI defined.
This commit is contained in:
Yann Collet
2017-10-18 17:01:53 -07:00
parent 9d377c6e88
commit d3b9547aa4
3 changed files with 33 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
programs/Makefile
View File

@@ -37,8 +37,7 @@ endif
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder \ -I$(ZSTDDIR)/dictBuilder \
-DXXH_NAMESPACE=ZSTD_ \ -DXXH_NAMESPACE=ZSTD_
-DZSTD_NEWAPI
CFLAGS ?= -O3 CFLAGS ?= -O3
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \

39
programs/bench.c
View File

@@ -287,7 +287,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
U32 nbLoops = 0; U32 nbLoops = 0;
ZSTD_CDict* cdict = NULL; ZSTD_CDict* cdict = NULL;
#ifdef ZSTD_NEWAPI
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads); ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag); ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
@@ -306,25 +305,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
#else
size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL };
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
#endif
do { do {
U32 blockNb; U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) { for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t rSize; size_t rSize;
#ifdef ZSTD_NEWAPI
ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 }; ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 };
ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 }; ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 };
size_t cError = 1; size_t cError = 1;
@@ -336,29 +320,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_getErrorName(cError)); ZSTD_getErrorName(cError));
} }
rSize = out.pos; rSize = out.pos;
#else /* ! ZSTD_NEWAPI */
if (dictBufferSize) {
rSize = ZSTD_compress_usingCDict(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cdict);
} else {
# ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
rSize = ZSTDMT_compressCCtx(mtctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cLevel);
# else
rSize = ZSTD_compress_advanced (ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
NULL, 0, zparams);
# endif
}
if (ZSTD_isError(rSize))
EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s",
ZSTD_getErrorName(rSize));
#endif /* ZSTD_NEWAPI */
blockTable[blockNb].cSize = rSize; blockTable[blockNb].cSize = rSize;
} }
nbLoops++; nbLoops++;

77
programs/fileio.c
View File

@@ -393,11 +393,7 @@ typedef struct {
size_t srcBufferSize; size_t srcBufferSize;
void* dstBuffer; void* dstBuffer;
size_t dstBufferSize; size_t dstBufferSize;
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
ZSTDMT_CCtx* cctx;
#else
ZSTD_CStream* cctx; ZSTD_CStream* cctx;
#endif
} cRess_t; } cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
@@ -406,24 +402,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
cRess_t ress; cRess_t ress;
memset(&ress, 0, sizeof(ress)); memset(&ress, 0, sizeof(ress));
#ifdef ZSTD_NEWAPI
ress.cctx = ZSTD_createCCtx(); ress.cctx = ZSTD_createCCtx();
if (ress.cctx == NULL) if (ress.cctx == NULL)
EXM_THROW(30, "allocation error : can't create ZSTD_CCtx"); EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
#elif defined(ZSTD_MULTITHREAD)
ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
if (ress.cctx == NULL)
EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx");
if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
/* use complete window for overlap */
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);
if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET)
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog);
#else
ress.cctx = ZSTD_createCStream();
if (ress.cctx == NULL)
EXM_THROW(30, "allocation error : can't create ZSTD_CStream");
#endif
ress.srcBufferSize = ZSTD_CStreamInSize(); ress.srcBufferSize = ZSTD_CStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize); ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_CStreamOutSize(); ress.dstBufferSize = ZSTD_CStreamOutSize();
@@ -437,9 +418,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
if (dictFileName && (dictBuffer==NULL)) if (dictFileName && (dictBuffer==NULL))
EXM_THROW(32, "allocation error : can't create dictBuffer"); EXM_THROW(32, "allocation error : can't create dictBuffer");
#ifdef ZSTD_NEWAPI CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size, when available (note: supposed to be default anyway) */
{ /* frame parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
/* compression level */ /* compression level */
@@ -470,35 +449,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */
}
#elif defined(ZSTD_MULTITHREAD)
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
}
#else
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
params.fParams.checksumFlag = g_checksumFlag;
params.fParams.noDictIDFlag = !g_dictIDFlag;
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
}
#endif
free(dictBuffer); free(dictBuffer);
} }
@@ -509,11 +460,7 @@ static void FIO_freeCResources(cRess_t ress)
{ {
free(ress.srcBuffer); free(ress.srcBuffer);
free(ress.dstBuffer); free(ress.dstBuffer);
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
ZSTDMT_freeCCtx(ress.cctx);
#else
ZSTD_freeCStream(ress.cctx); /* never fails */ ZSTD_freeCStream(ress.cctx); /* never fails */
#endif
} }
@@ -814,14 +761,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
} }
/* init */ /* init */
#ifdef ZSTD_NEWAPI if (fileSize != UTIL_FILESIZE_UNKNOWN)
if (fileSize!=UTIL_FILESIZE_UNKNOWN) /* when src is stdin, fileSize==0, but is effectively unknown */
ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize); ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
#elif defined(ZSTD_MULTITHREAD)
CHECK( ZSTDMT_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) );
#else
CHECK( ZSTD_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) );
#endif
/* Main compression loop */ /* Main compression loop */
while (1) { while (1) {
@@ -833,14 +774,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
while (inBuff.pos != inBuff.size) { while (inBuff.pos != inBuff.size) {
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
#ifdef ZSTD_NEWAPI
CHECK( ZSTD_compress_generic(ress.cctx, CHECK( ZSTD_compress_generic(ress.cctx,
&outBuff, &inBuff, ZSTD_e_continue) ); &outBuff, &inBuff, ZSTD_e_continue) );
#elif defined(ZSTD_MULTITHREAD)
CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) );
#else
CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) );
#endif
/* Write compressed stream */ /* Write compressed stream */
if (outBuff.pos) { if (outBuff.pos) {
@@ -871,15 +806,9 @@ static int FIO_compressFilename_internal(cRess_t ress,
{ size_t result = 1; { size_t result = 1;
while (result != 0) { while (result != 0) {
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
#ifdef ZSTD_NEWAPI
ZSTD_inBuffer inBuff = { NULL, 0, 0 }; ZSTD_inBuffer inBuff = { NULL, 0, 0 };
result = ZSTD_compress_generic(ress.cctx, result = ZSTD_compress_generic(ress.cctx,
&outBuff, &inBuff, ZSTD_e_end); &outBuff, &inBuff, ZSTD_e_end);
#elif defined(ZSTD_MULTITHREAD)
result = ZSTDMT_endStream(ress.cctx, &outBuff);
#else
result = ZSTD_endStream(ress.cctx, &outBuff);
#endif
if (ZSTD_isError(result)) { if (ZSTD_isError(result)) {
EXM_THROW(26, "Compression error during frame end : %s", EXM_THROW(26, "Compression error during frame end : %s",
ZSTD_getErrorName(result)); ZSTD_getErrorName(result));