lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-29 11:21:22 +03:00
Code Activity

created ZSTD_compress_usingCDict_advanced()

Browse Source
This commit is contained in:
Yann Collet
2017-04-27 11:31:55 -07:00
parent 69a54d138a
commit f4bd857d81
4 changed files with 50 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
doc/zstd_manual.html
View File

@ -427,12 +427,19 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
both values are optional, select `0` if unknown. both values are optional, select `0` if unknown.
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, <pre><b>size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const void* dict,size_t dictSize, const void* dict,size_t dictSize,
ZSTD_parameters params); ZSTD_parameters params);
</b><p> Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter </b><p> Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter
</p></pre><BR>
<pre><b>size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
</b><p> Same as ZSTD_compress_usingDict_advanced(), with fine-tune control over frame parameters
</p></pre><BR> </p></pre><BR>
<a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre> <a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre>

12
lib/compress/zstd_compress.c
View File

@ -3047,6 +3047,15 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0); return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
} }
size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
{
CHECK_F (ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
}
/*! ZSTD_compress_usingCDict() : /*! ZSTD_compress_usingCDict() :
* Compression using a digested Dictionary. * Compression using a digested Dictionary.
* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
@ -3058,8 +3067,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
const ZSTD_CDict* cdict) const ZSTD_CDict* cdict)
{ {
ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
CHECK_F (ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */ return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }

19
lib/zstd.h
View File

@ -514,12 +514,19 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
/*! ZSTD_compress_advanced() : /*! ZSTD_compress_advanced() :
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ * Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter */
ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const void* dict,size_t dictSize, const void* dict,size_t dictSize,
ZSTD_parameters params); ZSTD_parameters params);
/*! ZSTD_compress_usingCDict_advanced() :
* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict, ZSTD_frameParameters fParams);
/*--- Advanced decompression functions ---*/ /*--- Advanced decompression functions ---*/

28
tests/fuzzer.c
View File

@ -405,9 +405,9 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++); DISPLAYLEVEL(4, "test%3i : compress with preprocessed dictionary : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, dictSize); { ZSTD_compressionParameters cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
{ ZSTD_customMem customMem = { NULL, NULL, NULL }; { ZSTD_customMem customMem = { NULL, NULL, NULL };
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, params.cParams, customMem); ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 1, cParams, customMem);
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize), cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, cdict); CNBuffer, CNBuffSize, cdict);
ZSTD_freeCDict(cdict); ZSTD_freeCDict(cdict);
@ -760,7 +760,6 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) { for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
size_t sampleSize, maxTestSize, totalTestSize; size_t sampleSize, maxTestSize, totalTestSize;
size_t cSize, totalCSize, totalGenSize; size_t cSize, totalCSize, totalGenSize;
XXH64_state_t xxhState;
U64 crcOrig; U64 crcOrig;
BYTE* sampleBuffer; BYTE* sampleBuffer;
const BYTE* dict; const BYTE* dict;
@ -917,22 +916,22 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
} else { } else {
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize); ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
ZSTD_parameters p; ZSTD_parameters const p = { cPar, fPar };
size_t errorCode; size_t const errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
p.cParams = cPar; p.fParams = fpar;
errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode)); CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
} }
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0); { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
} } } }
XXH64_reset(&xxhState, 0);
ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1); ZSTD_setCCtxParameter(ctx, ZSTD_p_forceWindow, FUZ_rand(&lseed) & 1);
{ U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
U32 n; U32 n;
XXH64_state_t xxhState;
XXH64_reset(&xxhState, 0);
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize); size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
@ -947,13 +946,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
totalTestSize += segmentSize; totalTestSize += segmentSize;
} } }
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
cSize += flushResult; cSize += flushResult;
}
crcOrig = XXH64_digest(&xxhState);
} }
crcOrig = XXH64_digest(&xxhState);
/* streaming decompression test */ /* streaming decompression test */
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */