From ac175d46d4815388f46c6726278f6e4a7d2c4f8d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 13 Sep 2016 00:51:47 +0200 Subject: [PATCH 01/22] updated comments --- lib/zstd.h | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index 5cc40c63c..a8b2de124 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -48,7 +48,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); * Simple API ***************************************/ /*! ZSTD_compress() : - Compresses `src` buffer into already allocated `dst`. + Compresses `src` content as a single zstd compressed frame into already allocated `dst`. Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. @return : the number of bytes written into `dst` (<= `dstCapacity), or an error code if it fails (which can be tested using ZSTD_isError()) */ @@ -56,32 +56,33 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); -/*! ZSTD_getDecompressedSize() : -* @return : decompressed size as a 64-bits value _if known_, 0 otherwise. -* note 1 : decompressed size can be very large (64-bits value), -* potentially larger than what local system can handle as a single memory segment. -* In which case, it's necessary to use streaming mode to decompress data. -* note 2 : decompressed size is an optional field, that may not be present. -* When `return==0`, data to decompress can have any size. -* In which case, it's necessary to use streaming mode to decompress data. -* Optionally, application may rely on its own implied limits. -* (For example, application data could be necessarily cut into blocks <= 16 KB). -* note 3 : decompressed size could be wrong or intentionally modified ! -* Always ensure result fits within application's authorized limits ! -* Each application can set its own limits. -* note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ -ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); - /*! ZSTD_decompress() : - `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ). - If originalSize is unknown, and if there is no implied application-specific limitations, - it's preferable to use streaming mode to decompress data. + `compressedSize` : must be the _exact_ size of a single compressed frame. + `dstCapacity` is an upper bound of originalSize. + If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), or an errorCode if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +/*! ZSTD_getDecompressedSize() : +* 'src' is the start of a zstd compressed frame. +* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. +* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. +* When `return==0`, data to decompress could be any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application can still use ZSTD_decompress() while relying on implied limits. +* (For example, data may be necessarily cut into blocks <= 16 KB). +* note 2 : decompressed size is always present when compression is done with ZSTD_compress() +* note 3 : decompressed size can be very large (64-bits value), +* potentially larger than what local system can handle as a single memory segment. +* In which case, it's necessary to use streaming mode to decompress data. +* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. +* Always ensure result fits within application's authorized limits. +* Each application can set its own limits. +* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + /*====== Helper functions ======*/ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ From 26ec25406666a689b0995c11b88c6fe38cf46a48 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 13 Sep 2016 16:52:16 +0200 Subject: [PATCH 02/22] new strategy for faster DDict decompression --- lib/decompress/zstd_decompress.c | 109 +++++++++++++++++++------------ 1 file changed, 67 insertions(+), 42 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index c6bb5329c..67c9a6d23 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -80,8 +80,12 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, struct ZSTD_DCtx_s { + const FSE_DTable* LLTptr; + const FSE_DTable* MLTptr; + const FSE_DTable* OFTptr; + const HUF_DTable* HUFptr; FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; - FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; + FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ const void* previousDstEnd; @@ -119,11 +123,15 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->base = NULL; dctx->vBase = NULL; dctx->dictEnd = NULL; - dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); + dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue)); - memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); + memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->LLTable; + dctx->MLTptr = dctx->MLTable; + dctx->OFTptr = dctx->OFTable; + dctx->HUFptr = dctx->hufTable; return 0; } @@ -159,6 +167,25 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ } +void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + ZSTD_decompressBegin(dstDCtx); + dstDCtx->dictEnd = srcDCtx->dictEnd; + dstDCtx->vBase = srcDCtx->vBase; + dstDCtx->base = srcDCtx->base; + dstDCtx->previousDstEnd = srcDCtx->previousDstEnd; + dstDCtx->dictID = srcDCtx->dictID; + dstDCtx->litEntropy = srcDCtx->litEntropy; + dstDCtx->fseEntropy = srcDCtx->fseEntropy; + dstDCtx->LLTptr = srcDCtx->LLTable; + dstDCtx->MLTptr = srcDCtx->MLTable; + dstDCtx->OFTptr = srcDCtx->OFTable; + dstDCtx->HUFptr = srcDCtx->hufTable; + dstDCtx->rep[0] = srcDCtx->rep[0]; + dstDCtx->rep[1] = srcDCtx->rep[1]; + dstDCtx->rep[2] = srcDCtx->rep[2]; +} + /*-************************************************************* * Decompression section @@ -350,34 +377,31 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, { case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ /* 2 - 2 - 10 - 10 */ - { singleStream = !lhlCode; - lhSize = 3; - litSize = (lhc >> 4) & 0x3FF; - litCSize = (lhc >> 14) & 0x3FF; - break; - } + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; case 2: /* 2 - 2 - 14 - 14 */ - { lhSize = 4; - litSize = (lhc >> 4) & 0x3FFF; - litCSize = lhc >> 18; - break; - } + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; case 3: /* 2 - 2 - 18 - 18 */ - { lhSize = 5; - litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + (istart[4] << 10); - break; - } + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + (istart[4] << 10); + break; } if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); if (HUF_isError((litEncType==set_repeat) ? ( singleStream ? - HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) : - HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) : + HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) : + HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) : ( singleStream ? HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) @@ -387,6 +411,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable; return litCSize + lhSize; } @@ -461,7 +486,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, @return : nb bytes read from src, or an error code if it fails, testable with ZSTD_isError() */ -FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog, +static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog, const void* src, size_t srcSize, const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable) { @@ -491,8 +516,7 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e } -size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, - FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 flagRepeatTable, +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, const void* src, size_t srcSize) { const BYTE* const istart = (const BYTE* const)src; @@ -522,16 +546,19 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, ip++; /* Build DTables */ - { size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); + { size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + if (LLtype != set_repeat) dctx->LLTptr = dctx->LLTable; ip += llhSize; } - { size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + if (OFtype != set_repeat) dctx->OFTptr = dctx->OFTable; ip += ofhSize; } - { size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + if (MLtype != set_repeat) dctx->MLTptr = dctx->MLTable; ip += mlhSize; } } @@ -714,16 +741,13 @@ static size_t ZSTD_decompressSequences( const BYTE* litPtr = dctx->litPtr; const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; const BYTE* const litEnd = litPtr + dctx->litSize; - FSE_DTable* DTableLL = dctx->LLTable; - FSE_DTable* DTableML = dctx->MLTable; - FSE_DTable* DTableOffb = dctx->OffTable; const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); int nbSeq; /* Build Decoding Tables */ - { size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize); + { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); if (ZSTD_isError(seqHSize)) return seqHSize; ip += seqHSize; } @@ -733,10 +757,10 @@ static size_t ZSTD_decompressSequences( seqState_t seqState; dctx->fseEntropy = 1; { U32 i; for (i=0; irep[i]; } - CHECK_E(BIT_initDStream(&(seqState.DStream), ip, iend-ip), corruption_detected); - FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); - FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); - FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { nbSeq--; @@ -895,15 +919,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, /*! ZSTD_decompress_usingPreparedDCtx() : -* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. +* Same as ZSTD_decompress_usingDict(), but using a reference context `refDCtx`, where dictionary has been loaded. * It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). +* `refDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). * Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx, +static size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - ZSTD_copyDCtx(dctx, refDCtx); + //ZSTD_copyDCtx(dctx, refDCtx); + ZSTD_refDCtx(dctx, refDCtx); ZSTD_checkContinuity(dctx, dst); return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); } @@ -911,8 +936,8 @@ size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDC size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict, size_t dictSize) + const void* src, size_t srcSize, + const void* dict, size_t dictSize) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); @@ -1120,7 +1145,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); + CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); dictPtr += offcodeHeaderSize; } From 30d305615a8836af6646da0d752ba54d3693363a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 13 Sep 2016 17:23:31 +0200 Subject: [PATCH 03/22] updated NEWS --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 167666c4a..b6a1d6983 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,7 @@ Fixed : CLI -d output to stdout by default when input is stdin (#322) Fixed : CLI correctly detects console on Mac OS-X Fixed : Legacy decoders use unified error codes (#341), reported by benrg Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) +Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) Fixed : zstd-pgo, reported by octoploid (#329) v1.0.0 From 0be21d790a0851c0e14626a093adf33990a1a2ee Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 13 Sep 2016 17:33:47 +0200 Subject: [PATCH 04/22] fixed fullbench --- tests/fullbench.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index eaf1fc774..670b51681 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -121,13 +121,12 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co } extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); -extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 tableRepeatFlag, const void* src, size_t srcSize); +extern size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeq, const void* src, size_t srcSize); size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) { - U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */ int nbSeq; (void)src; (void)srcSize; (void)dst; (void)dstSize; - return ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, 0, buff2, g_cSize); + return ZSTD_decodeSeqHeaders(g_zdc, &nbSeq, buff2, g_cSize); } @@ -289,6 +288,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) } iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ ip += ZSTD_blockHeaderSize; /* skip block header */ + ZSTD_decompressBegin(g_zdc); ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */ g_cSize = iend-ip; memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ From c4cc9bf9735ed9dccff88c5acc87ccfb06c325ae Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 13 Sep 2016 17:50:08 +0200 Subject: [PATCH 05/22] -r generates an error on systems which do not support it --- programs/zstdcli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e829c93d6..66b75a199 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -334,8 +334,10 @@ int main(int argCount, char** argv) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; argument++; break; +#ifdef UTIL_HAS_CREATEFILELIST /* recursive */ case 'r': recursive=1; argument++; break; +#endif #ifndef ZSTD_NOBENCH /* Benchmark */ From 64a84edef56b03e104b572b5e57f8f6a1e1c1bb0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 13 Sep 2016 17:54:37 +0200 Subject: [PATCH 06/22] added -r support for Mac OS-X --- programs/util.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/util.h b/programs/util.h index e0d1e536c..b281a3acb 100644 --- a/programs/util.h +++ b/programs/util.h @@ -262,7 +262,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ return nbFiles; } -#elif (defined(__unix__) || defined(__unix) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) /* snprintf, opendir */ +#elif (defined(__APPLE__) && defined(__MACH__)) || \ + ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ # include From 220c567aa15c98f0ed7dd2ae85862699091844aa Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 13 Sep 2016 19:40:50 +0200 Subject: [PATCH 07/22] updated NEWS --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index b6a1d6983..ace6d4826 100644 --- a/NEWS +++ b/NEWS @@ -4,7 +4,8 @@ added : NetBSD install target (#338) Improved : variable compression speed improvements on batches of small files. Fixed : CLI -d output to stdout by default when input is stdin (#322) Fixed : CLI correctly detects console on Mac OS-X -Fixed : Legacy decoders use unified error codes (#341), reported by benrg +Fixed : CLI supports recursive mode `-r` on Mac OS-X +Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) Fixed : zstd-pgo, reported by octoploid (#329) From 64deef3bee9a5235191ff31d446be6f1e591a4b2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 14 Sep 2016 00:16:07 +0200 Subject: [PATCH 08/22] Fixed srcSize=1 --- lib/compress/zstd_compress.c | 2 +- lib/zstd.h | 2 +- tests/fuzzer.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f832e081a..1d59279e3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -168,7 +168,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1< srcLog) cPar.windowLog = srcLog; } } if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; diff --git a/lib/zstd.h b/lib/zstd.h index a8b2de124..6e3d43596 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -50,7 +50,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); /*! ZSTD_compress() : Compresses `src` content as a single zstd compressed frame into already allocated `dst`. Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - @return : the number of bytes written into `dst` (<= `dstCapacity), + @return : compressed size written into `dst` (<= `dstCapacity), or an error code if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 323854b32..b8f102a9c 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -57,7 +57,7 @@ static U32 g_displayLevel = 2; if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayClock = clock(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stdout); } } -static const clock_t g_refreshRate = CLOCKS_PER_SEC * 150 / 1000; +static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6; static clock_t g_displayClock = 0; From d092d77cfc03c4807ce2a873ca0d9bce13352418 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 14 Sep 2016 16:14:57 +0200 Subject: [PATCH 09/22] minor variable renaming --- lib/decompress/zstd_decompress.c | 43 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 67c9a6d23..0840e7f94 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -167,9 +167,9 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ } -void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) { - ZSTD_decompressBegin(dstDCtx); + ZSTD_decompressBegin(dstDCtx); /* init */ dstDCtx->dictEnd = srcDCtx->dictEnd; dstDCtx->vBase = srcDCtx->vBase; dstDCtx->base = srcDCtx->base; @@ -927,7 +927,6 @@ static size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - //ZSTD_copyDCtx(dctx, refDCtx); ZSTD_refDCtx(dctx, refDCtx); ZSTD_checkContinuity(dctx, dst); return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); @@ -1290,7 +1289,7 @@ typedef enum { zdss_init, zdss_loadHeader, /* *** Resource management *** */ struct ZSTD_DStream_s { - ZSTD_DCtx* zd; + ZSTD_DCtx* dctx; ZSTD_frameParams fParams; ZSTD_dStreamStage stage; char* inBuff; @@ -1302,7 +1301,7 @@ struct ZSTD_DStream_s { size_t outStart; size_t outEnd; size_t blockSize; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ size_t lhSize; ZSTD_customMem customMem; void* dictContent; @@ -1331,8 +1330,8 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) if (zds==NULL) return NULL; memset(zds, 0, sizeof(ZSTD_DStream)); memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); - zds->zd = ZSTD_createDCtx_advanced(customMem); - if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } + zds->dctx = ZSTD_createDCtx_advanced(customMem); + if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; } zds->stage = zdss_init; zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; return zds; @@ -1342,7 +1341,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) { if (zds==NULL) return 0; /* support free on null */ { ZSTD_customMem const cMem = zds->customMem; - ZSTD_freeDCtx(zds->zd); + ZSTD_freeDCtx(zds->dctx); ZSTD_free(zds->inBuff, cMem); ZSTD_free(zds->outBuff, cMem); ZSTD_free(zds->dictContent, cMem); @@ -1398,7 +1397,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { - return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize + zds->dictSize; + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + zds->inBuffSize + zds->outBuffSize + zds->dictSize; } @@ -1462,11 +1461,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } /* Consume header */ - ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize); - { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_prefix */ - CHECK_F(ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size)); - { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd); - CHECK_F(ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size)); + ZSTD_decompressBegin_usingDict(zds->dctx, zds->dictContent, zds->dictSize); + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); + { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size)); } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); @@ -1492,15 +1491,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* pass-through */ case zdss_read: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); if (neededInSize==0) { /* end of frame */ zds->stage = zdss_init; someMoreWork = 0; break; } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ - const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); - size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -1516,7 +1515,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } case zdss_load: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ size_t loadedSize; if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ @@ -1526,8 +1525,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ - { const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); - size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + { const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, zds->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -1559,7 +1558,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* result */ input->pos += (size_t)(ip-istart); output->pos += (size_t)(op-ostart); - { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); if (!nextSrcSizeHint) { /* frame fully decoded */ if (zds->outEnd == zds->outStart) { /* output fully flushed */ if (zds->hostageByte) { @@ -1574,7 +1573,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } return 1; } - nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); /* preload header of next block */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ nextSrcSizeHint -= zds->inPos; /* already loaded*/ return nextSrcSizeHint; From e91c4b4cef41e13207474d8c28b7043c0c090a75 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 14 Sep 2016 16:55:44 +0200 Subject: [PATCH 10/22] introduced ZSTD_resetDStream() . added : ZSTD_sizeof_DDict() --- lib/decompress/zstd_decompress.c | 71 ++++++++++++++------------------ lib/zstd.h | 7 +++- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 0840e7f94..1ea330bed 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -918,21 +918,6 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } -/*! ZSTD_decompress_usingPreparedDCtx() : -* Same as ZSTD_decompress_usingDict(), but using a reference context `refDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `refDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -static size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - ZSTD_refDCtx(dctx, refDCtx); - ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); -} - - size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1196,7 +1181,6 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict return ZSTD_refDictContent(dctx, dict, dictSize); } - size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { CHECK_F(ZSTD_decompressBegin(dctx)); @@ -1205,6 +1189,8 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t } +/* ====== ZSTD_DDict ====== */ + struct ZSTD_DDict_s { void* dict; size_t dictSize; @@ -1263,20 +1249,26 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict) } } +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize; +} + + /*! ZSTD_decompress_usingDDict() : * Decompression using a pre-digested Dictionary * Use dictionary without significant overhead. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict) +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); #endif - return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, - dst, dstCapacity, - src, srcSize); + ZSTD_refDCtx(dctx, ddict->refContext); + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); } @@ -1290,6 +1282,7 @@ typedef enum { zdss_init, zdss_loadHeader, /* *** Resource management *** */ struct ZSTD_DStream_s { ZSTD_DCtx* dctx; + ZSTD_DDict* ddict; ZSTD_frameParams fParams; ZSTD_dStreamStage stage; char* inBuff; @@ -1304,9 +1297,6 @@ struct ZSTD_DStream_s { BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ size_t lhSize; ZSTD_customMem customMem; - void* dictContent; - size_t dictSize; - const void* dictSource; void* legacyContext; U32 previousLegacyVersion; U32 legacyVersion; @@ -1342,9 +1332,9 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) if (zds==NULL) return 0; /* support free on null */ { ZSTD_customMem const cMem = zds->customMem; ZSTD_freeDCtx(zds->dctx); + ZSTD_freeDDict(zds->ddict); ZSTD_free(zds->inBuff, cMem); ZSTD_free(zds->outBuff, cMem); - ZSTD_free(zds->dictContent, cMem); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (zds->legacyContext) ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); @@ -1364,15 +1354,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di { zds->stage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) { /* new dictionary */ - if (dictSize > zds->dictSize) { - ZSTD_free(zds->dictContent, zds->customMem); - zds->dictContent = ZSTD_malloc(dictSize, zds->customMem); - if (zds->dictContent == NULL) return ERROR(memory_allocation); - } - memcpy(zds->dictContent, dict, dictSize); - zds->dictSize = dictSize; - } + ZSTD_freeDDict(zds->ddict); + zds->ddict = ZSTD_createDDict(dict, dictSize); + if (zds->ddict == NULL) return ERROR(memory_allocation); zds->legacyVersion = 0; zds->hostageByte = 0; return ZSTD_frameHeaderSize_prefix; @@ -1383,6 +1367,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } +size_t ZSTD_resetDStream(ZSTD_DStream* zds) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + zds->legacyVersion = 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; +} + size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue) { @@ -1397,7 +1390,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { - return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + zds->inBuffSize + zds->outBuffSize + zds->dictSize; + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize; } @@ -1439,7 +1432,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); if (legacyVersion) { CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, - zds->dictContent, zds->dictSize)); + zds->ddict->dict, zds->ddict->dictSize)); zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); } else { @@ -1461,7 +1454,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } /* Consume header */ - ZSTD_decompressBegin_usingDict(zds->dctx, zds->dictContent, zds->dictSize); + ZSTD_refDCtx(zds->dctx, zds->ddict->refContext); { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); diff --git a/lib/zstd.h b/lib/zstd.h index 6e3d43596..985c6cd4b 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -403,6 +403,10 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); * Gives the amount of memory used by a given ZSTD_DCtx */ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +/*! ZSTD_sizeof_DDict() : + * Gives the amount of memory used by a given ZSTD_DDict */ +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + /* ****************************************************************** * Advanced Streaming functions @@ -413,7 +417,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); + ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); @@ -423,6 +427,7 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); From 3ecbe6a37c82365138cb093dabf95313477fa96e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 14 Sep 2016 17:26:59 +0200 Subject: [PATCH 11/22] fileio uses ZSTD_resetDStream() --- programs/fileio.c | 12 +++++++----- tests/playTests.sh | 2 +- tests/zstreamtest.c | 12 ++++++++---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 056958245..598848d34 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -480,8 +480,6 @@ typedef struct { size_t srcBufferSize; void* dstBuffer; size_t dstBufferSize; - void* dictBuffer; - size_t dictBufferSize; ZSTD_DStream* dctx; FILE* dstFile; } dRess_t; @@ -501,7 +499,12 @@ static dRess_t FIO_createDResources(const char* dictFileName) if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); /* dictionary */ - ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName); + { void* dictBuffer; + size_t const dictBufferSize = FIO_loadFile(&dictBuffer, dictFileName); + size_t const initError = ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize); + if (ZSTD_isError(initError)) EXM_THROW(61, "ZSTD_initDStream_usingDict error : %s", ZSTD_getErrorName(initError)); + free(dictBuffer); + } return ress; } @@ -512,7 +515,6 @@ static void FIO_freeDResources(dRess_t ress) if (ZSTD_isError(errorCode)) EXM_THROW(69, "Error : can't free ZSTD_DStream context resource : %s", ZSTD_getErrorName(errorCode)); free(ress.srcBuffer); free(ress.dstBuffer); - free(ress.dictBuffer); } @@ -601,7 +603,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, size_t readSize; U32 storedSkips = 0; - ZSTD_initDStream_usingDict(ress.dctx, ress.dictBuffer, ress.dictBufferSize); + ZSTD_resetDStream(ress.dctx); /* Header loading (optional, saves one loop) */ { size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */ diff --git a/tests/playTests.sh b/tests/playTests.sh index 21e98bf90..042197c2d 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -205,7 +205,7 @@ $MD5SUM dirTestDict/* > tmph1 $ZSTD -f --rm dirTestDict/* -D tmpDictC $ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default if [[ "$OSTYPE" == "darwin"* ]]; then - $ECHO "test skipped on OS-X" # not compatible with OS-X's md5 + $ECHO "md5sum -c not supported on OS-X : test skipped" # not compatible with OS-X's md5 else $MD5SUM -c tmph1 fi diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 97fbaa18e..d10d4f125 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -209,7 +209,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Byte-by-byte decompression test */ DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); - { size_t r = 1; + { /* skippable frame */ + size_t r = 1; ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); inBuff.src = compressedBuffer; outBuff.dst = decodedBuffer; @@ -221,9 +222,10 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (ZSTD_isError(r)) goto _output_error; } + /* normal frame */ ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); r=1; - while (r) { /* normal frame */ + while (r) { inBuff.size = inBuff.pos + 1; outBuff.size = outBuff.pos + 1; r = ZSTD_decompressStream(zd, &outBuff, &inBuff); @@ -322,6 +324,8 @@ _output_error: } +/* ====== Fuzzer tests ====== */ + static size_t findDiff(const void* buf1, const void* buf2, size_t max) { const BYTE* b1 = (const BYTE*)buf1; @@ -413,8 +417,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; - /* states full reset (unsynchronized) */ - /* some issues only happen when reusing states in a specific sequence of parameters */ + /* states full reset (deliberately not synchronized) */ + /* some issues can only happen when reusing states */ if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); } if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); } From 6fb4d675c642989020e00e33899df1cb21029874 Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Wed, 14 Sep 2016 19:01:04 +0200 Subject: [PATCH 12/22] add FSE decoding tables for predefined distributions to spec They can so serve as a sample result of the table construction algorithm. --- zstd_compression_format.md | 182 +++++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index b14f55534..f350cdabe 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -1164,6 +1164,188 @@ __`Content`__ : The rest of the dictionary is its content. [compressed blocks]: #the-format-of-compressed_block +Appendix A - Decoding tables for predefined codes +------------------------------------------------- + +This appendix contains FSE decoding tables for the predefined literal length, match length, and offset +codes. The tables have been constructed using the algorithm as given above in the +"from normalized distribution to decoding tables" chapter. The tables here can be used as examples +to crosscheck that an implementation implements the decoding table generation algorithm correctly. + +#### Literal Length Code: + +| State | Symbol | Number_Of_Bits | Base | +| ----- | ------ | -------------- | ---- | +| 0 | 0 | 4 | 0 | +| 1 | 0 | 4 | 16 | +| 2 | 1 | 5 | 32 | +| 3 | 3 | 5 | 0 | +| 4 | 4 | 5 | 0 | +| 5 | 6 | 5 | 0 | +| 6 | 7 | 5 | 0 | +| 7 | 9 | 5 | 0 | +| 8 | 10 | 5 | 0 | +| 9 | 12 | 5 | 0 | +| 10 | 14 | 6 | 0 | +| 11 | 16 | 5 | 0 | +| 12 | 18 | 5 | 0 | +| 13 | 19 | 5 | 0 | +| 14 | 21 | 5 | 0 | +| 15 | 22 | 5 | 0 | +| 16 | 24 | 5 | 0 | +| 17 | 25 | 5 | 32 | +| 18 | 26 | 5 | 0 | +| 19 | 27 | 6 | 0 | +| 20 | 29 | 6 | 0 | +| 21 | 31 | 6 | 0 | +| 22 | 0 | 4 | 32 | +| 23 | 1 | 4 | 0 | +| 24 | 2 | 5 | 0 | +| 25 | 4 | 5 | 32 | +| 26 | 5 | 5 | 0 | +| 27 | 7 | 5 | 32 | +| 28 | 8 | 5 | 0 | +| 29 | 10 | 5 | 32 | +| 30 | 11 | 5 | 0 | +| 31 | 13 | 6 | 0 | +| 32 | 16 | 5 | 32 | +| 33 | 17 | 5 | 0 | +| 34 | 19 | 5 | 32 | +| 35 | 20 | 5 | 0 | +| 36 | 22 | 5 | 32 | +| 37 | 23 | 5 | 0 | +| 38 | 25 | 4 | 0 | +| 39 | 25 | 4 | 16 | +| 40 | 26 | 5 | 32 | +| 41 | 28 | 6 | 0 | +| 42 | 30 | 6 | 0 | +| 43 | 0 | 4 | 48 | +| 44 | 1 | 4 | 16 | +| 45 | 2 | 5 | 32 | +| 46 | 3 | 5 | 32 | +| 47 | 5 | 5 | 32 | +| 48 | 6 | 5 | 32 | +| 49 | 8 | 5 | 32 | +| 50 | 9 | 5 | 32 | +| 51 | 11 | 5 | 32 | +| 52 | 12 | 5 | 32 | +| 53 | 15 | 6 | 0 | +| 54 | 17 | 5 | 32 | +| 55 | 18 | 5 | 32 | +| 56 | 20 | 5 | 32 | +| 57 | 21 | 5 | 32 | +| 58 | 23 | 5 | 32 | +| 59 | 24 | 5 | 32 | +| 60 | 35 | 6 | 0 | +| 61 | 34 | 6 | 0 | +| 62 | 33 | 6 | 0 | +| 63 | 32 | 6 | 0 | + +#### Match Length Code: + +| State | Symbol | Number_Of_Bits | Base | +| ----- | ------ | -------------- | ---- | +| 0 | 0 | 6 | 0 | +| 1 | 1 | 4 | 0 | +| 2 | 2 | 5 | 32 | +| 3 | 3 | 5 | 0 | +| 4 | 5 | 5 | 0 | +| 5 | 6 | 5 | 0 | +| 6 | 8 | 5 | 0 | +| 7 | 10 | 6 | 0 | +| 8 | 13 | 6 | 0 | +| 9 | 16 | 6 | 0 | +| 10 | 19 | 6 | 0 | +| 11 | 22 | 6 | 0 | +| 12 | 25 | 6 | 0 | +| 13 | 28 | 6 | 0 | +| 14 | 31 | 6 | 0 | +| 15 | 33 | 6 | 0 | +| 16 | 35 | 6 | 0 | +| 17 | 37 | 6 | 0 | +| 18 | 39 | 6 | 0 | +| 19 | 41 | 6 | 0 | +| 20 | 43 | 6 | 0 | +| 21 | 45 | 6 | 0 | +| 22 | 1 | 4 | 16 | +| 23 | 2 | 4 | 0 | +| 24 | 3 | 5 | 32 | +| 25 | 4 | 5 | 0 | +| 26 | 6 | 5 | 32 | +| 27 | 7 | 5 | 0 | +| 28 | 9 | 6 | 0 | +| 29 | 12 | 6 | 0 | +| 30 | 15 | 6 | 0 | +| 31 | 18 | 6 | 0 | +| 32 | 21 | 6 | 0 | +| 33 | 24 | 6 | 0 | +| 34 | 27 | 6 | 0 | +| 35 | 30 | 6 | 0 | +| 36 | 32 | 6 | 0 | +| 37 | 34 | 6 | 0 | +| 38 | 36 | 6 | 0 | +| 39 | 38 | 6 | 0 | +| 40 | 40 | 6 | 0 | +| 41 | 42 | 6 | 0 | +| 42 | 44 | 6 | 0 | +| 43 | 1 | 4 | 32 | +| 44 | 1 | 4 | 48 | +| 45 | 2 | 4 | 16 | +| 46 | 4 | 5 | 32 | +| 47 | 5 | 5 | 32 | +| 48 | 7 | 5 | 32 | +| 49 | 8 | 5 | 32 | +| 50 | 11 | 6 | 0 | +| 51 | 14 | 6 | 0 | +| 52 | 17 | 6 | 0 | +| 53 | 20 | 6 | 0 | +| 54 | 23 | 6 | 0 | +| 55 | 26 | 6 | 0 | +| 56 | 29 | 6 | 0 | +| 57 | 52 | 6 | 0 | +| 58 | 51 | 6 | 0 | +| 59 | 50 | 6 | 0 | +| 60 | 49 | 6 | 0 | +| 61 | 48 | 6 | 0 | +| 62 | 47 | 6 | 0 | +| 63 | 46 | 6 | 0 | + +#### Offset Code: + +| State | Symbol | Number_Of_Bits | Base | +| ----- | ------ | -------------- | ---- | +| 0 | 0 | 5 | 0 | +| 1 | 6 | 4 | 0 | +| 2 | 9 | 5 | 0 | +| 3 | 15 | 5 | 0 | +| 4 | 21 | 5 | 0 | +| 5 | 3 | 5 | 0 | +| 6 | 7 | 4 | 0 | +| 7 | 12 | 5 | 0 | +| 8 | 18 | 5 | 0 | +| 9 | 23 | 5 | 0 | +| 10 | 5 | 5 | 0 | +| 11 | 8 | 4 | 0 | +| 12 | 14 | 5 | 0 | +| 13 | 20 | 5 | 0 | +| 14 | 2 | 5 | 0 | +| 15 | 7 | 4 | 16 | +| 16 | 11 | 5 | 0 | +| 17 | 17 | 5 | 0 | +| 18 | 22 | 5 | 0 | +| 19 | 4 | 5 | 0 | +| 20 | 8 | 4 | 16 | +| 21 | 13 | 5 | 0 | +| 22 | 19 | 5 | 0 | +| 23 | 1 | 5 | 0 | +| 24 | 6 | 4 | 16 | +| 25 | 10 | 5 | 0 | +| 26 | 16 | 5 | 0 | +| 27 | 28 | 5 | 0 | +| 28 | 27 | 5 | 0 | +| 29 | 26 | 5 | 0 | +| 30 | 25 | 5 | 0 | +| 31 | 24 | 5 | 0 | Version changes --------------- From 35ad602c26eefb3bc1a68e33ac019578ba6fee98 Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Wed, 14 Sep 2016 19:14:49 +0200 Subject: [PATCH 13/22] spec: clarify how bitstream exactly needs to be reversed for reading --- zstd_compression_format.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index f350cdabe..025439f38 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -1049,15 +1049,23 @@ by reading the required `Number_of_Bits`, and adding the specified `Baseline`. #### Bitstream -All sequences are stored in a single bitstream, read _backward_. -It is therefore necessary to know the bitstream size, -which is deducted from compressed block size. +FSE bitstreams are read in reverse direction than written. In zstd, +the compressor writes bits forward into a block and the decompressor +must read the bitstream _backwards_. -The last useful bit of the stream is followed by an end-bit-flag. -Highest bit of last byte is this flag. -It does not belong to the useful part of the bitstream. -Therefore, last byte has 0-7 useful bits. -Note that it also means that last byte cannot be `0`. +To find the start of the bitstream it is therefore necessary to +know the offset of the last byte of the block which can be found +by counting `Block_Size` bytes after the block header. + +After writing the last bit containing information, the compressor +writes a single `1`-bit and then fills the byte with 0-7 `0` bits of +padding. The last byte of the compressed bitstream cannot be `0` for +that reason. + +When decompressing, the last byte containing the padding is the first +byte to read. The decompressor needs to skip 0-7 initial `0`-bits and +the first `1`-bit it occurs. Afterwards, the useful part of the bitstream +begins. ##### Starting states From 55981a9ad61a1cc13422e58868a9b54fdc1a481d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Sep 2016 02:13:18 +0200 Subject: [PATCH 14/22] updated format doc version --- zstd_compression_format.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 025439f38..b58b43f5a 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -16,7 +16,7 @@ Distribution of this document is unlimited. ### Version -0.2.0 (22/07/16) +0.2.2 (14/09/16) Introduction @@ -1357,6 +1357,7 @@ to crosscheck that an implementation implements the decoding table generation al Version changes --------------- +- 0.2.2 : added predefined codes, by Johannes Rudolph - 0.2.1 : clarify field names, by Przemyslaw Skibinski - 0.2.0 : numerous format adjustments for zstd v0.8 - 0.1.2 : limit Huffman tree depth to 11 bits From d7c6589df8bfd0722579054532a0c588b9c17901 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Sep 2016 02:50:27 +0200 Subject: [PATCH 15/22] support ZSTD_sizeof_*() on NULL added ZSTD_sizeof_CDict() --- lib/compress/zstd_compress.c | 8 ++++++++ lib/decompress/zstd_decompress.c | 4 +++- lib/zstd.h | 12 ++++++++---- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1d59279e3..1b380d3fe 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -113,6 +113,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { + if (cctx==NULL) return 0; /* support sizeof on NULL */ return sizeof(*cctx) + cctx->workSpaceSize; } @@ -2680,6 +2681,12 @@ struct ZSTD_CDict_s { ZSTD_CCtx* refContext; }; /* typedef'd tp ZSTD_CDict within "zstd.h" */ +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize; +} + ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) { if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; @@ -2862,6 +2869,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) { + if (zcs==NULL) return 0; /* support sizeof on NULL */ return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 1ea330bed..2b2539a4a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -111,7 +111,7 @@ struct ZSTD_DCtx_s BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ -size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; return sizeof(ZSTD_DCtx); } /* support sizeof on NULL */ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } @@ -1251,6 +1251,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict) size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) { + if (ddict==NULL) return 0; /* support sizeof on NULL */ return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize; } @@ -1390,6 +1391,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { + if (zds==NULL) return 0; /* support sizeof on NULL */ return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize; } diff --git a/lib/zstd.h b/lib/zstd.h index 985c6cd4b..93d8d212e 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -352,14 +352,18 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_sizeofCCtx() : + * Gives the amount of memory used by a given ZSTD_CCtx */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); + /*! ZSTD_createCDict_advanced() : * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem); -/*! ZSTD_sizeofCCtx() : - * Gives the amount of memory used by a given ZSTD_CCtx */ -ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +/*! ZSTD_sizeof_CDict() : + * Gives the amount of memory used by a given ZSTD_sizeof_CDict */ +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. @@ -399,7 +403,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -/*! ZSTD_sizeofDCtx() : +/*! ZSTD_sizeof_DCtx() : * Gives the amount of memory used by a given ZSTD_DCtx */ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); From fa0c09760cc3891e1618834151afc5a7256dbfd0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Sep 2016 14:11:01 +0200 Subject: [PATCH 16/22] variable renaming --- lib/compress/zstd_compress.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1b380d3fe..611a4758a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2767,7 +2767,8 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; struct ZSTD_CStream_s { - ZSTD_CCtx* zc; + ZSTD_CCtx* cctx; + //ZSTD_CDict* cdict; char* inBuff; size_t inBuffSize; size_t inToCompress; @@ -2800,8 +2801,8 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) if (zcs==NULL) return NULL; memset(zcs, 0, sizeof(ZSTD_CStream)); memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); - zcs->zc = ZSTD_createCCtx_advanced(customMem); - if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; } + zcs->cctx = ZSTD_createCCtx_advanced(customMem); + if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; } return zcs; } @@ -2809,7 +2810,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) { if (zcs==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = zcs->customMem; - ZSTD_freeCCtx(zcs->zc); + ZSTD_freeCCtx(zcs->cctx); ZSTD_free(zcs->inBuff, cMem); ZSTD_free(zcs->outBuff, cMem); ZSTD_free(zcs, cMem); @@ -2844,7 +2845,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, if (zcs->outBuff == NULL) return ERROR(memory_allocation); } - CHECK_F(ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize)); + CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, dict, dictSize, params, pledgedSrcSize)); zcs->inToCompress = 0; zcs->inBuffPos = 0; @@ -2870,7 +2871,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) { if (zcs==NULL) return 0; /* support sizeof on NULL */ - return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize; + return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + zcs->outBuffSize + zcs->inBuffSize; } /*====== Compression ======*/ @@ -2921,8 +2922,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, else cDst = zcs->outBuff, oSize = zcs->outBuffSize; cSize = (flush == zsf_end) ? - ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : - ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); + ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); if (ZSTD_isError(cSize)) return cSize; if (flush == zsf_end) zcs->frameEnded = 1; /* prepare next block */ @@ -3016,7 +3017,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) /* create epilogue */ zcs->stage = zcss_final; zcs->outBuffContentSize = !notEnded ? 0 : - ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ + ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ } /* flush epilogue */ From 4cb212938cff82d0c0d70e67d97934cda2ffb901 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Sep 2016 14:54:07 +0200 Subject: [PATCH 17/22] introduced ZSTD_resetCStream() --- lib/compress/zstd_compress.c | 52 +++++++++++++++++++++++------------- lib/zstd.h | 5 ++-- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 611a4758a..f2cbbc229 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -239,9 +239,9 @@ typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_comp note : 'params' must be validated */ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ZSTD_parameters params, U64 frameContentSize, - ZSTD_compResetPolicy_e crp) + ZSTD_compResetPolicy_e const crp) { - if (crp == ZSTDcrp_continue) /* still some issues */ + if (crp == ZSTDcrp_continue) if (ZSTD_equivalentParams(params, zc->params)) return ZSTD_continueCCtx(zc, params, frameContentSize); @@ -2738,17 +2738,23 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) } } +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) +{ + if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext)) + else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize)); + return 0; +} + /*! ZSTD_compress_usingCDict() : * Compression using a digested Dictionary. * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. * Note that compression level is decided during dictionary creation */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict) +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) { - if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext)) - else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, srcSize)); + CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); if (cdict->refContext->params.fParams.contentSizeFlag==1) { cctx->params.fParams.contentSizeFlag = 1; @@ -2768,7 +2774,7 @@ typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; struct ZSTD_CStream_s { ZSTD_CCtx* cctx; - //ZSTD_CDict* cdict; + ZSTD_CDict* cdict; char* inBuff; size_t inBuffSize; size_t inToCompress; @@ -2824,6 +2830,19 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, U64 pledgedSrcSize) +{ + CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)); + + zcs->inToCompress = 0; + zcs->inBuffPos = 0; + zcs->inBuffTarget = zcs->blockSize; + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + zcs->frameEnded = 0; + return 0; /* ready to go */ +} + size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) @@ -2845,16 +2864,13 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, if (zcs->outBuff == NULL) return ERROR(memory_allocation); } - CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, dict, dictSize, params, pledgedSrcSize)); + ZSTD_freeCDict(zcs->cdict); + zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); + if (zcs->cdict == NULL) return ERROR(memory_allocation); - zcs->inToCompress = 0; - zcs->inBuffPos = 0; - zcs->inBuffTarget = zcs->blockSize; - zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_load; zcs->checksum = params.fParams.checksumFlag > 0; - zcs->frameEnded = 0; - return 0; /* ready to go */ + + return ZSTD_resetCStream(zcs, pledgedSrcSize); } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) @@ -2871,7 +2887,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) { if (zcs==NULL) return 0; /* support sizeof on NULL */ - return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + zcs->outBuffSize + zcs->inBuffSize; + return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize; } /*====== Compression ======*/ diff --git a/lib/zstd.h b/lib/zstd.h index 93d8d212e..f79a5dcac 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -421,7 +421,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); + ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ +ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); @@ -431,8 +432,8 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); +ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); From 43eeea47253a1c75cbc65de26cfc5e978537b9b2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Sep 2016 15:38:44 +0200 Subject: [PATCH 18/22] fileio uses ZSTD_resetCStream() --- programs/fileio.c | 73 +++++++++++++++++++++------------------------- programs/fileio.h | 1 - programs/zstdcli.c | 2 +- 3 files changed, 34 insertions(+), 42 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 598848d34..7dee7c11b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -119,8 +119,6 @@ static clock_t g_time = 0; ***************************************/ static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } -static U32 g_maxWLog = 23; -void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; } static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } static U32 g_dictIDFlag = 1; @@ -167,7 +165,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName) return f; } -/* `dstFileName must` be non-NULL */ +/** FIO_openDstFile() : + * condition : `dstFileName` must be non-NULL. + * @result : FILE* to `dstFileName`, or NULL if it fails */ static FILE* FIO_openDstFile(const char* dstFileName) { FILE* f; @@ -250,14 +250,12 @@ typedef struct { size_t srcBufferSize; void* dstBuffer; size_t dstBufferSize; - void* dictBuffer; - size_t dictBufferSize; ZSTD_CStream* cctx; FILE* dstFile; FILE* srcFile; } cRess_t; -static cRess_t FIO_createCResources(const char* dictFileName) +static cRess_t FIO_createCResources(const char* dictFileName, int cLevel) { cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -271,19 +269,27 @@ static cRess_t FIO_createCResources(const char* dictFileName) if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "zstd: allocation error : not enough memory"); /* dictionary */ - ress.dictBufferSize = FIO_loadFile(&(ress.dictBuffer), dictFileName); + { void* dictBuffer; + size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName); + if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer"); + { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictBuffSize); + params.fParams.contentSizeFlag = 1; + params.fParams.checksumFlag = g_checksumFlag; + params.fParams.noDictIDFlag = !g_dictIDFlag; + { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, 0); + if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); + } } + free(dictBuffer); + } return ress; } static void FIO_freeCResources(cRess_t ress) { - size_t errorCode; free(ress.srcBuffer); free(ress.dstBuffer); - free(ress.dictBuffer); - errorCode = ZSTD_freeCStream(ress.cctx); - if (ZSTD_isError(errorCode)) EXM_THROW(38, "zstd: error : can't release ZSTD_CStream : %s", ZSTD_getErrorName(errorCode)); + ZSTD_freeCStream(ress.cctx); /* never fails */ } @@ -293,8 +299,7 @@ static void FIO_freeCResources(cRess_t ress) * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_internal(cRess_t ress, - const char* dstFileName, const char* srcFileName, - int cLevel) + const char* dstFileName, const char* srcFileName) { FILE* const srcFile = ress.srcFile; FILE* const dstFile = ress.dstFile; @@ -303,17 +308,9 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 const fileSize = UTIL_getFileSize(srcFileName); /* init */ - { ZSTD_parameters params = ZSTD_getParams(cLevel, fileSize, ress.dictBufferSize); - params.fParams.contentSizeFlag = 1; - params.fParams.checksumFlag = g_checksumFlag; - params.fParams.noDictIDFlag = !g_dictIDFlag; - if ((g_maxWLog) && (params.cParams.windowLog > g_maxWLog)) { - params.cParams.windowLog = g_maxWLog; - params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize); - } - { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); - if (ZSTD_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(errorCode)); - } } + { size_t const resetError = ZSTD_resetCStream(ress.cctx, fileSize); + if (ZSTD_isError(resetError)) EXM_THROW(21, "Error initializing compression : %s", ZSTD_getErrorName(resetError)); + } /* Main compression loop */ while (1) { @@ -367,8 +364,7 @@ static int FIO_compressFilename_internal(cRess_t ress, * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_srcFile(cRess_t ress, - const char* dstFileName, const char* srcFileName, - int cLevel) + const char* dstFileName, const char* srcFileName) { int result; @@ -380,10 +376,10 @@ static int FIO_compressFilename_srcFile(cRess_t ress, ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); + result = FIO_compressFilename_internal(ress, dstFileName, srcFileName); fclose(ress.srcFile); - if ((g_removeSrcFile) && (!result)) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } + if (g_removeSrcFile && !result) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ return result; } @@ -393,17 +389,16 @@ static int FIO_compressFilename_srcFile(cRess_t ress, * 1 : pb */ static int FIO_compressFilename_dstFile(cRess_t ress, - const char* dstFileName, const char* srcFileName, - int cLevel) + const char* dstFileName, const char* srcFileName) { int result; ress.dstFile = FIO_openDstFile(dstFileName); - if (ress.dstFile==0) return 1; + if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ - result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel); + result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName); - if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } + if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */ if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */ return result; } @@ -414,8 +409,8 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, { clock_t const start = clock(); - cRess_t const ress = FIO_createCResources(dictFileName); - int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); + cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel); + int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); @@ -433,7 +428,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile size_t dfnSize = FNSPACE; char* dstFileName = (char*)malloc(FNSPACE); size_t const suffixSize = suffix ? strlen(suffix) : 0; - cRess_t ress = FIO_createCResources(dictFileName); + cRess_t ress = FIO_createCResources(dictFileName, compressionLevel); /* init */ if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); @@ -445,8 +440,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile ress.dstFile = stdout; SET_BINARY_MODE(stdout); for (u=0; u Date: Thu, 15 Sep 2016 16:16:21 +0200 Subject: [PATCH 19/22] fixed minor conversion warning --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f2cbbc229..df8201beb 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2830,7 +2830,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } -size_t ZSTD_resetCStream(ZSTD_CStream* zcs, U64 pledgedSrcSize) +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)); From 3e47dbcc8c0e5d37715212f69e9e076f0b4b62e3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Sep 2016 17:00:02 +0200 Subject: [PATCH 20/22] fixed memory leak --- tests/Makefile | 2 +- tests/datagencli.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 3ce9f317e..17cf6589e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -154,7 +154,7 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly)) HOST_OS = POSIX -valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 +valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 valgrindTest: zstd datagen fuzzer fullbench zbufftest @echo "\n ---- valgrind tests : memory analyzer ----" $(VALGRIND) ./datagen -g50M > $(VOID) diff --git a/tests/datagencli.c b/tests/datagencli.c index 772e3dc99..2f3ebc4d6 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -9,7 +9,7 @@ /*-************************************ -* Includes +* Dependencies **************************************/ #include "util.h" /* Compiler options */ #include /* fprintf, stderr */ From a6bdf55759a9b28cd20d10434604625f91b9b7b9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Sep 2016 17:02:06 +0200 Subject: [PATCH 21/22] fixed memory leak --- lib/compress/zstd_compress.c | 1 + programs/datagen.c | 5 ++--- programs/fileio.h | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index df8201beb..f5b712fc3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2817,6 +2817,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) if (zcs==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = zcs->customMem; ZSTD_freeCCtx(zcs->cctx); + ZSTD_freeCDict(zcs->cdict); ZSTD_free(zcs->inBuff, cMem); ZSTD_free(zcs->outBuff, cMem); ZSTD_free(zcs, cMem); diff --git a/programs/datagen.c b/programs/datagen.c index 109b8e3d8..1af5a38a5 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -20,10 +20,9 @@ /*-************************************ * Dependencies **************************************/ -#include /* malloc */ +#include /* malloc, free */ #include /* FILE, fwrite, fprintf */ #include /* memcpy */ -#include /* errno */ #include "mem.h" /* U32 */ @@ -176,7 +175,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba, BYTE ldt[LTSIZE]; /* literals distribution table */ /* init */ - if (buff==NULL) { fprintf(stderr, "datagen: error: %s \n", strerror(errno)); exit(1); } + if (buff==NULL) { perror("datagen"); exit(1); } if (litProba<=0.0) litProba = matchProba / 4.5; memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */ RDG_fillLiteralDistrib(ldt, litProba); diff --git a/programs/fileio.h b/programs/fileio.h index 9cf6c96fc..1e89aec27 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -8,7 +8,8 @@ */ -#pragma once +#ifndef FILEIO_H_23981798732 +#define FILEIO_H_23981798732 #if defined (__cplusplus) extern "C" { @@ -69,3 +70,5 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles #if defined (__cplusplus) } #endif + +#endif /* FILEIO_H_23981798732 */ From 55f276949c8eea6158acd51973135d9fb7fa4e59 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 15 Sep 2016 17:23:15 +0200 Subject: [PATCH 22/22] removed option unsupported by travis --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 17cf6589e..3ce9f317e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -154,7 +154,7 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly)) HOST_OS = POSIX -valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 +valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1 valgrindTest: zstd datagen fuzzer fullbench zbufftest @echo "\n ---- valgrind tests : memory analyzer ----" $(VALGRIND) ./datagen -g50M > $(VOID)