diff --git a/lib/zstd_buffered.c b/lib/zstd_buffered.c index 48721d628..aab83e606 100644 --- a/lib/zstd_buffered.c +++ b/lib/zstd_buffered.c @@ -335,8 +335,6 @@ struct ZBUFF_DCtx_s { size_t outStart; size_t outEnd; size_t hPos; - const char* dict; - size_t dictSize; ZBUFF_dStage stage; unsigned char headerBuffer[ZSTD_frameHeaderSize_max]; }; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ @@ -365,19 +363,16 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc) /* *** Initialization *** */ -size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) +size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbc, const void* dict, size_t dictSize) { zbc->stage = ZBUFFds_readHeader; - zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = zbc->dictSize = 0; - return ZSTD_resetDCtx(zbc->zc); + zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0; + return ZSTD_decompressBegin_usingDict(zbc->zc, dict, dictSize); } - -size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* zbc, const void* src, size_t srcSize) +size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc) { - zbc->dict = (const char*)src; - zbc->dictSize = srcSize; - return 0; + return ZBUFF_decompressInitDictionary(zbc, NULL, 0); } @@ -458,8 +453,6 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt if (zbc->outBuff == NULL) return ERROR(memory_allocation); } } - if (zbc->dictSize) - ZSTD_decompress_insertDictionary(zbc->zc, zbc->dict, zbc->dictSize); if (zbc->hPos) { /* some data already loaded into headerBuffer : transfer into inBuff */ diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index d2316a82b..63101a107 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -48,7 +48,7 @@ extern "C" { /* *************************************************************** -* Tuning parameters +* Compiler specifics *****************************************************************/ /*! * ZSTD_DLL_EXPORT : @@ -69,7 +69,7 @@ ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); -ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* cctx, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZBUFF_compressWithDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr); @@ -119,7 +119,7 @@ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); -ZSTDLIB_API size_t ZBUFF_decompressWithDictionary(ZBUFF_DCtx* dctx, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); @@ -128,19 +128,16 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* * * A ZBUFF_DCtx object is required to track streaming operation. * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation. -* ZBUFF_DCtx objects can be reused multiple times. -* -* Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary() -* It must be the same content as the one set during compression phase. -* Dictionary content must remain accessible during the decompression process. +* Use ZBUFF_decompressInit() to start a new decompression operation, +* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFF_DCtx objects can be reused multiple times. * * Use ZBUFF_decompressContinue() repetitively to consume your input. * *srcSizePtr and *maxDstSizePtr can be any size. * The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. * The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) * or 0 when a frame is completely decoded * or an error code, which can be tested using ZBUFF_isError(). * @@ -157,7 +154,7 @@ ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); /** The below functions provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are not compulsory, they just tend to offer better latency */ +* These sizes are just hints, and tend to offer better latency */ ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index db01c3385..4124e4f2b 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -225,6 +225,42 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, } +/*! ZSTD_copyCCtx +* Duplicate an existing context @srcCCtx into another one @dstCCtx. +* Only works during stage 0 (i.e. before first call to ZSTD_compressContinue()) +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) +{ + const U32 contentLog = (srcCCtx->params.strategy == ZSTD_fast) ? 1 : srcCCtx->params.contentLog; + const size_t tableSpace = ((1 << contentLog) + (1 << srcCCtx->params.hashLog)) * sizeof(U32); + + if (srcCCtx->stage!=0) return ERROR(stage_wrong); + + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params); + + /* copy tables */ + memcpy(dstCCtx->hashTable, srcCCtx->hashTable, tableSpace); + + /* copy frame header */ + dstCCtx->hbSize = srcCCtx->hbSize; + memcpy(dstCCtx->headerBuffer , srcCCtx->headerBuffer, srcCCtx->hbSize); + + /* copy dictionary pointers */ + dstCCtx->nextToUpdate= srcCCtx->nextToUpdate; + dstCCtx->nextSrc = srcCCtx->nextSrc; + dstCCtx->base = srcCCtx->base; + dstCCtx->dictBase = srcCCtx->dictBase; + dstCCtx->dictLimit = srcCCtx->dictLimit; + dstCCtx->lowLimit = srcCCtx->lowLimit; + + dstCCtx->flagHufTable = srcCCtx->flagHufTable; + if (dstCCtx->flagHufTable) + memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4); + + return 0; +} + + /** ZSTD_reduceIndex * rescale indexes to avoid future overflow (indexes are U32) */ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, @@ -2133,49 +2169,17 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize) { - U32 magic = MEM_readLE32(dict); - U32 eSize; - if (magic != ZSTD_DICT_MAGIC) - return ZSTD_loadDictionaryContent(zc, dict, dictSize); - - eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4, dictSize-4) + 4; - if (ZSTD_isError(eSize)) return eSize; - return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); -} - - -/*! ZSTD_duplicateCCtx -* Duplicate an existing context @srcCCtx into another one @dstCCtx. -* Only works during stage 0 (i.e. before first call to ZSTD_compressContinue()) -* @return : 0, or an error code */ -size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) -{ - const U32 contentLog = (srcCCtx->params.strategy == ZSTD_fast) ? 1 : srcCCtx->params.contentLog; - const size_t tableSpace = ((1 << contentLog) + (1 << srcCCtx->params.hashLog)) * sizeof(U32); - - if (srcCCtx->stage!=0) return ERROR(stage_wrong); - - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params); - - /* copy tables */ - memcpy(dstCCtx->hashTable, srcCCtx->hashTable, tableSpace); - - /* copy frame header */ - dstCCtx->hbSize = srcCCtx->hbSize; - memcpy(dstCCtx->headerBuffer , srcCCtx->headerBuffer, srcCCtx->hbSize); - - /* copy dictionary pointers */ - dstCCtx->nextToUpdate= srcCCtx->nextToUpdate; - dstCCtx->nextSrc = srcCCtx->nextSrc; - dstCCtx->base = srcCCtx->base; - dstCCtx->dictBase = srcCCtx->dictBase; - dstCCtx->dictLimit = srcCCtx->dictLimit; - dstCCtx->lowLimit = srcCCtx->lowLimit; - - dstCCtx->flagHufTable = srcCCtx->flagHufTable; - if (dstCCtx->flagHufTable) - memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4); + if (dict && dictSize) + { + U32 magic = MEM_readLE32(dict); + size_t eSize; + if (magic != ZSTD_DICT_MAGIC) + return ZSTD_loadDictionaryContent(zc, dict, dictSize); + eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4, dictSize-4) + 4; + if (ZSTD_isError(eSize)) return eSize; + return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); + } return 0; } diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index ce54d200d..93e82884c 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -136,6 +136,7 @@ struct ZSTD_DCtx_s ZSTD_parameters params; blockType_t bType; ZSTD_dStage stage; + U32 flagHufTable; const BYTE* litPtr; size_t litBufSize; size_t litSize; @@ -143,7 +144,7 @@ struct ZSTD_DCtx_s BYTE headerBuffer[ZSTD_frameHeaderSize_max]; }; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */ -size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx) +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) { dctx->expected = ZSTD_frameHeaderSize_min; dctx->stage = ZSTDds_getFrameHeaderSize; @@ -152,6 +153,7 @@ size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx) dctx->vBase = NULL; dctx->dictEnd = NULL; dctx->hufTableX4[0] = HufLog; + dctx->flagHufTable = 0; return 0; } @@ -159,7 +161,7 @@ ZSTD_DCtx* ZSTD_createDCtx(void) { ZSTD_DCtx* dctx = (ZSTD_DCtx*)malloc(sizeof(ZSTD_DCtx)); if (dctx==NULL) return NULL; - ZSTD_resetDCtx(dctx); + ZSTD_decompressBegin(dctx); return dctx; } @@ -169,6 +171,12 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) return 0; } +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + memcpy(dstDCtx, srcDCtx, + sizeof(ZSTD_DCtx) - (BLOCKSIZE+WILDCOPY_OVERLENGTH + ZSTD_frameHeaderSize_max)); /* no need to copy workspace */ +} + /* ************************************************************* * Decompression section @@ -342,6 +350,8 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, U32 lhSize = ((istart[0]) >> 4) & 3; if (lhSize != 1) /* only case supported for now : small litSize, single stream */ return ERROR(corruption_detected); + if (!dctx->flagHufTable) + return ERROR(dictionary_corrupted); /* 2 - 2 - 10 - 10 */ lhSize=3; @@ -631,14 +641,12 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, *litPtr = litEnd; /* update for next sequence */ /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) - { + if (sequence.offset > (size_t)(oLitEnd - base)) { /* offset beyond prefix */ if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); match = dictEnd - (base-match); - if (match + sequence.matchLength <= dictEnd) - { + if (match + sequence.matchLength <= dictEnd) { memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; } @@ -653,8 +661,7 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, } /* match within prefix */ - if (sequence.offset < 8) - { + if (sequence.offset < 8) { /* close range match, overlap */ const int sub2 = dec64table[sequence.offset]; op[0] = match[0]; @@ -665,24 +672,20 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, ZSTD_copy4(op+4, match); match -= sub2; } - else - { + else { ZSTD_copy8(op, match); } op += 8; match += 8; - if (oMatchEnd > oend-12) - { - if (op < oend_8) - { + if (oMatchEnd > oend-12) { + if (op < oend_8) { ZSTD_wildcopy(op, match, oend_8 - op); match += oend_8 - op; op = oend_8; } while (op < oMatchEnd) *op++ = *match++; } - else - { + else { ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ } return sequenceLength; @@ -735,8 +738,7 @@ static size_t ZSTD_decompressSequences( FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); - for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) - { + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { size_t oneSeqSize; nbSeq--; ZSTD_decodeSequence(&sequence, &seqState); @@ -764,8 +766,7 @@ static size_t ZSTD_decompressSequences( static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) { - if (dst != dctx->previousDstEnd) /* not contiguous */ - { + if (dst != dctx->previousDstEnd) { /* not contiguous */ dctx->dictEnd = dctx->previousDstEnd; dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); dctx->base = dst; @@ -800,10 +801,11 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } -size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, +/*! ZSTD_decompress_continueDCtx +* dctx must have been properly initialized */ +static size_t ZSTD_decompress_continueDCtx(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* src, size_t srcSize, - const void* dict, size_t dictSize) + const void* src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; const BYTE* iend = ip + srcSize; @@ -813,21 +815,6 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, size_t remainingSize = srcSize; blockProperties_t blockProperties; - /* init */ - ZSTD_resetDCtx(dctx); - if (dict) - { - size_t errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize); - if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted); - dctx->dictEnd = dctx->previousDstEnd; - dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); - dctx->base = dst; - } - else - { - dctx->vBase = dctx->base = dctx->dictEnd = dst; - } - /* Frame Header */ { size_t frameHeaderSize; @@ -888,6 +875,27 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, } +size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx, + void* dst, size_t maxDstSize, + const void* src, size_t srcSize) +{ + ZSTD_copyDCtx(dctx, refDCtx); + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); +} + + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompress_continueDCtx(dctx, dst, maxDstSize, src, srcSize); +} + + size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) { return ZSTD_decompress_usingDict(dctx, dst, maxDstSize, src, srcSize, NULL, 0); @@ -933,8 +941,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co dctx->headerSize = ZSTD_decodeFrameHeader_Part1(dctx, src, ZSTD_frameHeaderSize_min); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); - if (dctx->headerSize > ZSTD_frameHeaderSize_min) - { + if (dctx->headerSize > ZSTD_frameHeaderSize_min) { dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; dctx->stage = ZSTDds_decodeFrameHeader; return 0; @@ -958,13 +965,11 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co blockProperties_t bp; size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(blockSize)) return blockSize; - if (bp.blockType == bt_end) - { + if (bp.blockType == bt_end) { dctx->expected = 0; dctx->stage = ZSTDds_getFrameHeaderSize; } - else - { + else { dctx->expected = blockSize; dctx->bType = bp.blockType; dctx->stage = ZSTDds_decompressBlock; @@ -990,7 +995,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co rSize = 0; break; default: - return ERROR(GENERIC); + return ERROR(GENERIC); /* impossible */ } dctx->stage = ZSTDds_decodeBlockHeader; dctx->expected = ZSTD_blockHeaderSize; @@ -1011,15 +1016,15 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi dctx->previousDstEnd = (const char*)dict + dictSize; } - static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { size_t hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize); if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); + dctx->flagHufTable = 1; return hSize; } -size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { size_t eSize; U32 magic = MEM_readLE32(dict); @@ -1042,3 +1047,18 @@ size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_ return 0; } + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + size_t errorCode; + errorCode = ZSTD_decompressBegin(dctx); + if (ZSTD_isError(errorCode)) return errorCode; + + if (dict && dictSize) { + errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize); + if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted); + } + + return 0; +} + diff --git a/lib/zstd_static.h b/lib/zstd_static.h index f99893d71..fd4131ae7 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -92,7 +92,7 @@ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint ZSTDLIB_API void ZSTD_validateParams(ZSTD_parameters* params); /** ZSTD_compress_usingDict -* Same as ZSTD_compressCCtx(), using a Dictionary content as prefix +* Same as ZSTD_compressCCtx(), loading a Dictionary content. * Note : dict can be NULL, in which case, it's equivalent to ZSTD_compressCCtx() */ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, @@ -108,14 +108,24 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params); -/** ZSTD_decompress_usingDict +/*! ZSTD_decompress_usingDict * Same as ZSTD_decompressDCtx, using a Dictionary content as prefix * Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */ -ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx, +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, const void* dict,size_t dictSize); +/*! ZSTD_decompress_usingPreparedDCtx +* Same as ZSTD_decompress_usingDict, but using a reference context preparedDCtx, where dictionary has already been loaded into. +* It avoids reloading the dictionary each time. +* preparedDCtx must have been properly initialized using ZSTD_compressBegin_usingDict(). +* Requires 2 contexts : 1 for reference, which will not be modified, and 1 to run the decompression operation */ +ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx( + ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx, + void* dst, size_t maxDstSize, + const void* src, size_t srcSize); + /* ************************************** * Streaming functions (direct mode) @@ -127,7 +137,7 @@ ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dic ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, ZSTD_parameters params); ZSTDLIB_API size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* ctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_duplicateCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx); +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize); @@ -164,9 +174,11 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSiz */ -ZSTDLIB_API size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx); + ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); @@ -176,20 +188,19 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma A ZSTD_DCtx object is required to track streaming operations. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. - A ZSTD_DCtx object can be re-used multiple times. Use ZSTD_resetDCtx() to return to fresh status. + A ZSTD_DCtx object can be re-used multiple times. - First operation is to retrieve frame parameters, using ZSTD_getFrameParams(). - This function doesn't consume its input. It needs enough input data to properly decode the frame header. + First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams(). + This operation is independent, and just needs enough input data to properly decode the frame header. Objective is to retrieve *params.windowlog, to know minimum amount of memory required during decoding. Result : 0 when successful, it means the ZSTD_parameters structure has been filled. >0 : means there is not enough data into src. Provides the expected size to successfully decode header. - errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header) + errorCode, which can be tested using ZSTD_isError() - Then, you can optionally insert a dictionary. - This operation must mimic the compressor behavior, otherwise decompression will fail or be corrupted. + Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict() + Alternatively, you can copy a prepared context, using ZSTD_copyDCtx() - Then it's possible to start decompression. - Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail. ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog). @@ -206,23 +217,23 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ma /* ************************************** * Block functions ****************************************/ - /*!Block functions produce and decode raw zstd blocks, without frame metadata. - It saves associated header sizes. - But user will have to save and regenerate fields required to regenerate data, such as block sizes. + Frame headers won't be generated. + User will have to save and regenerate fields required to regenerate data, such as block sizes. A few rules to respect : - Uncompressed block size must be <= 128 KB - - Compressing or decompressing require a context structure + - Compressing or decompressing requires a context structure + Use ZSTD_createXCtx() to create them - It is necessary to init context before starting - + compression : ZSTD_compressBegin(), which allows selection of compression level or parameters - + decompression : ZSTD_resetDCtx() - + If you compress multiple blocks without resetting, next blocks will create references to previous ones - - Dictionary can optionally be inserted, using ZSTD_de/compress_insertDictionary() + + compression : ZSTD_compressBegin() + + decompression : ZSTD_decompressBegin() + + variants _usingDict() are also allowed + + copyXCtx() works too - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. - + User must test for such outcome and be able to deal with uncompressed data - + ZSTD_decompressBlock() doesn't accept uncompressed data as input + In which case, nothing is produced into `dst`. + + User must test for such outcome and deal directly with uncompressed data + + ZSTD_decompressBlock() doesn't accept uncompressed data as input !! */ size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); diff --git a/programs/bench.c b/programs/bench.c index 30555660f..f5b6f8d5c 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -227,6 +227,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, void* const resultBuffer = malloc(srcSize); ZSTD_CCtx* refCtx = ZSTD_createCCtx(); ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* refDCtx = ZSTD_createDCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); U64 crcOrig = XXH64(srcBuffer, srcSize, 0); U32 nbBlocks = 0; @@ -235,7 +236,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ /* Memory allocation & restrictions */ - if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !dctx) + if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx) EXM_THROW(31, "not enough memory"); /* Init blockTable data */ @@ -298,7 +299,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_compress_insertDictionary(refCtx, dictBuffer, dictBufferSize); for (blockNb=0; blockNb%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); /* CRC Checking */ crcCheck = XXH64(resultBuffer, srcSize, 0); - if (crcOrig!=crcCheck) - { + if (crcOrig!=crcCheck) { size_t u; DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); - for (u=0; u u) break; bacc += blockTable[segNb].srcSize; } @@ -365,8 +367,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, bNb = pos / (128 KB); printf("(segment %u, block %u, pos %u) \n", segNb, bNb, pos); break; - } - } + } } break; } #endif @@ -375,7 +376,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (crcOrig == crcCheck) DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); else - DISPLAY("X \n"); + DISPLAY("\n"); } /* clean up */ @@ -383,6 +384,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, free(resultBuffer); ZSTD_freeCCtx(refCtx); ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(refDCtx); ZSTD_freeDCtx(dctx); return 0; } diff --git a/programs/fileio.c b/programs/fileio.c index 0d49af2b9..4bab75eae 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -529,8 +529,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, size_t readSize=alreadyLoaded; /* Main decompression Loop */ - ZBUFF_decompressInit(ress.dctx); - ZBUFF_decompressWithDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); + ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); while (1) { /* Decode */ diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 4058ef24d..b72f7e574 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -207,7 +207,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_isError(result)) goto _output_error; result = ZSTD_compress_insertDictionary(ctxOrig, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; - result = ZSTD_duplicateCCtx(ctxDuplicated, ctxOrig); + result = ZSTD_copyCCtx(ctxDuplicated, ctxOrig); if (ZSTD_isError(result)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); @@ -284,7 +284,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++); - result = ZSTD_resetDCtx(dctx); + result = ZSTD_decompressBegin(dctx); if (ZSTD_isError(result)) goto _output_error; result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; @@ -302,9 +302,8 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); - result = ZSTD_resetDCtx(dctx); + result = ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize); if (ZSTD_isError(result)) goto _output_error; - ZSTD_decompress_insertDictionary(dctx, CNBuffer, dictSize); result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize); if (ZSTD_isError(result)) goto _output_error; if (result != blockSize) goto _output_error; @@ -574,7 +573,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit CHECK (ZSTD_isError(errorCode), "start streaming error : %s", ZSTD_getErrorName(errorCode)); errorCode = ZSTD_compress_insertDictionary(refCtx, dict, dictSize); CHECK (ZSTD_isError(errorCode), "dictionary insertion error : %s", ZSTD_getErrorName(errorCode)); - errorCode = ZSTD_duplicateCCtx(ctx, refCtx); + errorCode = ZSTD_copyCCtx(ctx, refCtx); CHECK (ZSTD_isError(errorCode), "context duplication error : %s", ZSTD_getErrorName(errorCode)); totalTestSize = 0; cSize = 0; for (n=0; n