void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); +Return a CCtx to clean state. + Useful after an error, or to interrupt an ongoing compression job and start a new one. + Any internal data not yet flushed is cancelled. + Dictionary (if any) is dropped. + All parameters are back to default values. + It's possible to modify compression parameters after a reset. + +
typedef enum {
ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal conditions */
ZSTD_e_flush, /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
@@ -968,16 +978,6 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); -Return a CCtx to clean state. - Useful after an error, or to interrupt an ongoing compression job and start a new one. - Any internal data not yet flushed is cancelled. - Dictionary (if any) is dropped. - All parameters are back to default values. - It's possible to modify compression parameters after a reset. - -
size_t ZSTD_compress_generic_simpleArgs (
ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, size_t* dstPos,
@@ -992,6 +992,7 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);Quick howto : - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure - ZSTD_CCtxParam_setParameter() : Push parameters one by one into @@ -1010,18 +1011,18 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t
size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params); +size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);Reset params to default values.
-size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, int compressionLevel); +size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);Initializes the compression parameters of cctxParams according to compression level. All other parameters are reset to their default values.
-size_t ZSTD_initCCtxParams_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); +size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);Initializes the compression and frame parameters of cctxParams according to params. All other parameters are reset to their default values. diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 85678a8dc..528291777 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -143,28 +143,19 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) /* private API call, for dictBuilder only */ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } -static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - ZSTD_CCtx_params CCtxParams, U64 srcSizeHint, size_t dictSize) +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize) { - DEBUGLOG(5, "ZSTD_getCParamsFromCCtxParams: srcSize = %u, dictSize = %u", - (U32)srcSizeHint, (U32)dictSize); - return (CCtxParams.compressionLevel == ZSTD_CLEVEL_CUSTOM) ? - CCtxParams.cParams : - ZSTD_getCParams(CCtxParams.compressionLevel, srcSizeHint, dictSize); -} - -static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* CCtxParams, U64 srcSize) -{ - DEBUGLOG(4, "ZSTD_cLevelToCCtxParams_srcSize: srcSize = %u", - (U32)srcSize); - CCtxParams->cParams = ZSTD_getCParamsFromCCtxParams(*CCtxParams, srcSize, 0); - CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM; -} - -static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams) -{ - DEBUGLOG(4, "ZSTD_cLevelToCCtxParams"); - ZSTD_cLevelToCCtxParams_srcSize(CCtxParams, ZSTD_CONTENTSIZE_UNKNOWN); + ZSTD_compressionParameters cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize); + if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog; + if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog; + if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog; + if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog; + if (CCtxParams->cParams.searchLength) cParams.searchLength = CCtxParams->cParams.searchLength; + if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength; + if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy; + return cParams; } static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( @@ -173,7 +164,9 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( ZSTD_CCtx_params cctxParams; memset(&cctxParams, 0, sizeof(cctxParams)); cctxParams.cParams = cParams; - cctxParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; + cctxParams.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ + assert(!ZSTD_checkCParams(cParams)); + cctxParams.fParams.contentSizeFlag = 1; return cctxParams; } @@ -187,6 +180,7 @@ static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( if (!params) { return NULL; } params->customMem = customMem; params->compressionLevel = ZSTD_CLEVEL_DEFAULT; + params->fParams.contentSizeFlag = 1; return params; } @@ -202,36 +196,41 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) return 0; } -size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params) +size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) { - return ZSTD_initCCtxParams(params, ZSTD_CLEVEL_DEFAULT); + return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); } -size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, int compressionLevel) { +size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { if (!cctxParams) { return ERROR(GENERIC); } memset(cctxParams, 0, sizeof(*cctxParams)); cctxParams->compressionLevel = compressionLevel; + cctxParams->fParams.contentSizeFlag = 1; return 0; } -size_t ZSTD_initCCtxParams_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) +size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) { if (!cctxParams) { return ERROR(GENERIC); } CHECK_F( ZSTD_checkCParams(params.cParams) ); memset(cctxParams, 0, sizeof(*cctxParams)); cctxParams->cParams = params.cParams; cctxParams->fParams = params.fParams; - cctxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM; + cctxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ + assert(!ZSTD_checkCParams(params.cParams)); return 0; } +/* ZSTD_assignParamsToCCtxParams() : + * params is presumed valid at this stage */ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( ZSTD_CCtx_params cctxParams, ZSTD_parameters params) { ZSTD_CCtx_params ret = cctxParams; ret.cParams = params.cParams; ret.fParams = params.fParams; - ret.compressionLevel = ZSTD_CLEVEL_CUSTOM; + ret.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ + assert(!ZSTD_checkCParams(params.cParams)); return ret; } @@ -301,9 +300,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v case ZSTD_p_targetLength: case ZSTD_p_compressionStrategy: if (cctx->cdict) return ERROR(stage_wrong); - if (value>0) { - ZSTD_cLevelToCCtxParams_srcSize(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); /* Optimize cParams when srcSize is known */ - } return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_compressLiterals: @@ -328,11 +324,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); case ZSTD_p_enableLongDistanceMatching: - if (cctx->cdict) return ERROR(stage_wrong); - if (value>0) - ZSTD_cLevelToCCtxParams_srcSize(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); /* Optimize cParams when srcSize is known */ - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_ldmHashLog: case ZSTD_p_ldmMinMatch: case ZSTD_p_ldmBucketSizeLog: @@ -368,65 +359,50 @@ size_t ZSTD_CCtxParam_setParameter( } case ZSTD_p_windowLog : - DEBUGLOG(4, "ZSTD_CCtxParam_setParameter: set windowLog=%u", value); - if (value) { /* 0 : does not change current windowLog */ + if (value>0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - ZSTD_cLevelToCCtxParams(CCtxParams); - CCtxParams->cParams.windowLog = value; - } + CCtxParams->cParams.windowLog = value; return CCtxParams->cParams.windowLog; case ZSTD_p_hashLog : - if (value) { /* 0 : does not change current hashLog */ + if (value>0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - ZSTD_cLevelToCCtxParams(CCtxParams); - CCtxParams->cParams.hashLog = value; - } + CCtxParams->cParams.hashLog = value; return CCtxParams->cParams.hashLog; case ZSTD_p_chainLog : - if (value) { /* 0 : does not change current chainLog */ + if (value>0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - ZSTD_cLevelToCCtxParams(CCtxParams); - CCtxParams->cParams.chainLog = value; - } + CCtxParams->cParams.chainLog = value; return CCtxParams->cParams.chainLog; case ZSTD_p_searchLog : - if (value) { /* 0 : does not change current searchLog */ + if (value>0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - ZSTD_cLevelToCCtxParams(CCtxParams); - CCtxParams->cParams.searchLog = value; - } + CCtxParams->cParams.searchLog = value; return value; case ZSTD_p_minMatch : - if (value) { /* 0 : does not change current minMatch length */ + if (value>0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - ZSTD_cLevelToCCtxParams(CCtxParams); - CCtxParams->cParams.searchLength = value; - } + CCtxParams->cParams.searchLength = value; return CCtxParams->cParams.searchLength; case ZSTD_p_targetLength : - if (value) { /* 0 : does not change current sufficient_len */ + if (value>0) /* 0 => use default */ CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); - ZSTD_cLevelToCCtxParams(CCtxParams); - CCtxParams->cParams.targetLength = value; - } + CCtxParams->cParams.targetLength = value; return CCtxParams->cParams.targetLength; case ZSTD_p_compressionStrategy : - if (value) { /* 0 : does not change currentstrategy */ + if (value>0) /* 0 => use default */ CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra); - ZSTD_cLevelToCCtxParams(CCtxParams); - CCtxParams->cParams.strategy = (ZSTD_strategy)value; - } + CCtxParams->cParams.strategy = (ZSTD_strategy)value; return (size_t)CCtxParams->cParams.strategy; case ZSTD_p_compressLiterals: CCtxParams->disableLiteralCompression = !value; - return !!value; + return !CCtxParams->disableLiteralCompression; case ZSTD_p_contentSizeFlag : /* Content size written in frame header _when known_ (default:1) */ @@ -441,7 +417,7 @@ size_t ZSTD_CCtxParam_setParameter( case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ DEBUGLOG(4, "set dictIDFlag = %u", (value>0)); - CCtxParams->fParams.noDictIDFlag = (value == 0); + CCtxParams->fParams.noDictIDFlag = !value; return !CCtxParams->fParams.noDictIDFlag; case ZSTD_p_forceMaxWindow : @@ -450,7 +426,7 @@ size_t ZSTD_CCtxParam_setParameter( case ZSTD_p_nbWorkers : #ifndef ZSTD_MULTITHREAD - if (value > 0) return ERROR(parameter_unsupported); + if (value>0) return ERROR(parameter_unsupported); return 0; #else return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value); @@ -471,39 +447,32 @@ size_t ZSTD_CCtxParam_setParameter( #endif case ZSTD_p_enableLongDistanceMatching : - if (value) { - ZSTD_cLevelToCCtxParams(CCtxParams); - CCtxParams->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; - } - return ZSTD_ldm_initializeParameters(&CCtxParams->ldmParams, value); + CCtxParams->ldmParams.enableLdm = (value>0); + return CCtxParams->ldmParams.enableLdm; case ZSTD_p_ldmHashLog : - if (value) { /* 0 : does not change current ldmHashLog */ + if (value>0) /* 0 ==> auto */ CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - CCtxParams->ldmParams.hashLog = value; - } + CCtxParams->ldmParams.hashLog = value; return CCtxParams->ldmParams.hashLog; case ZSTD_p_ldmMinMatch : - if (value) { /* 0 : does not change current ldmMinMatch */ + if (value>0) /* 0 ==> default */ CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX); - CCtxParams->ldmParams.minMatchLength = value; - } + CCtxParams->ldmParams.minMatchLength = value; return CCtxParams->ldmParams.minMatchLength; case ZSTD_p_ldmBucketSizeLog : - if (value > ZSTD_LDM_BUCKETSIZELOG_MAX) { + if (value > ZSTD_LDM_BUCKETSIZELOG_MAX) return ERROR(parameter_outOfBound); - } CCtxParams->ldmParams.bucketSizeLog = value; - return value; + return CCtxParams->ldmParams.bucketSizeLog; case ZSTD_p_ldmHashEveryLog : - if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) { + if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) return ERROR(parameter_outOfBound); - } CCtxParams->ldmParams.hashEveryLog = value; - return value; + return CCtxParams->ldmParams.hashEveryLog; default: return ERROR(parameter_unsupported); } @@ -547,7 +516,7 @@ size_t ZSTD_CCtx_loadDictionary_advanced( cctx->cdict = NULL; } else { ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize); + ZSTD_getCParamsFromCCtxParams(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize); cctx->cdictLocal = ZSTD_createCDict_advanced( dict, dictSize, dictLoadMethod, dictContentType, @@ -720,14 +689,14 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) /* Estimate CCtx size is supported for single-threaded compression only. */ if (params->nbWorkers > 0) { return ERROR(GENERIC); } { ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(*params, 0, 0); + ZSTD_getCParamsFromCCtxParams(params, 0, 0); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.searchLength==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; size_t const tokenSpace = blockSize + 11*maxNbSeq; size_t const entropySpace = HUF_WORKSPACE_SIZE; size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); - size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms->cParams, /* forCCtx */ 1); + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams); size_t const ldmSeqSpace = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq); @@ -922,6 +891,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl cctx->dictID = 0; if (params.ldmParams.enableLdm) ZSTD_window_clear(&cctx->ldmState.window); + ZSTD_referenceExternalSequences(cctx, NULL, 0); ZSTD_invalidateMatchState(&cctx->blockState.matchState); ZSTD_reset_compressedBlockState(cctx->blockState.prevCBlock); XXH64_reset(&cctx->xxhState, 0); @@ -988,8 +958,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zbuff, pledgedSrcSize)) { DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%u)", zc->appliedParams.cParams.windowLog, (U32)zc->blockSize); - assert(!(params.ldmParams.enableLdm && - params.ldmParams.hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET)); return ZSTD_continueCCtx(zc, params, pledgedSrcSize); } } DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); @@ -1108,6 +1076,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ptr = zc->ldmState.bucketOffsets + ldmBucketSize; ZSTD_window_clear(&zc->ldmState.window); } + ZSTD_referenceExternalSequences(zc, NULL, 0); /* buffers */ zc->inBuffSize = buffInSize; @@ -1582,19 +1551,126 @@ size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, } } -#define FUNCTION(fn) fn##_default -#define TARGET -#include "zstd_compress_impl.h" -#undef TARGET -#undef FUNCTION +FORCE_INLINE_TEMPLATE size_t +ZSTD_encodeSequences_body( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + CHECK_E(BIT_initCStream(&blockStream, dst, dstCapacity), dstSize_tooSmall); /* not enough space remaining */ + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n
= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offset, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offset >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + } } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ + return streamSize; + } +} + +static size_t +ZSTD_encodeSequences_default( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + #if DYNAMIC_BMI2 -#define FUNCTION(fn) fn##_bmi2 -#define TARGET TARGET_ATTRIBUTE("bmi2") -#include "zstd_compress_impl.h" -#undef TARGET -#undef FUNCTION +static TARGET_ATTRIBUTE("bmi2") size_t +ZSTD_encodeSequences_bmi2( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} #endif @@ -1818,8 +1894,10 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, ZSTD_matchState_t* const ms = &zc->blockState.matchState; DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", (U32)dstCapacity, ms->window.dictLimit, ms->nextToUpdate); - if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength); return 0; /* don't even attempt compression below a certain srcSize */ + } ZSTD_resetSeqStore(&(zc->seqStore)); /* limited update after a very long match */ @@ -2097,8 +2175,8 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) { - ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(cctx->appliedParams, 0, 0); + ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; + assert(!ZSTD_checkCParams(cParams)); return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); } @@ -2541,7 +2619,8 @@ static size_t ZSTD_initCDict_internal( ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams) { - DEBUGLOG(3, "ZSTD_initCDict_internal, mode %u", (U32)dictContentType); + DEBUGLOG(3, "ZSTD_initCDict_internal, dictContentType %u", (U32)dictContentType); + assert(!ZSTD_checkCParams(cParams)); cdict->cParams = cParams; if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { cdict->dictBuffer = NULL; @@ -2557,8 +2636,7 @@ static size_t ZSTD_initCDict_internal( /* Reset the state to no dictionary */ ZSTD_reset_compressedBlockState(&cdict->cBlockState); - { - void* const end = ZSTD_reset_matchState( + { void* const end = ZSTD_reset_matchState( &cdict->matchState, (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, &cParams, ZSTDcrp_continue, /* forCCtx */ 0); @@ -2568,15 +2646,13 @@ static size_t ZSTD_initCDict_internal( /* (Maybe) load the dictionary * Skips loading the dictionary if it is <= 8 bytes. */ - { - ZSTD_CCtx_params params; + { ZSTD_CCtx_params params; memset(¶ms, 0, sizeof(params)); params.compressionLevel = ZSTD_CLEVEL_DEFAULT; params.fParams.contentSizeFlag = 1; params.cParams = cParams; - { - size_t const dictID = ZSTD_compress_insertDictionary( - &cdict->cBlockState, &cdict->matchState,¶ms, + { size_t const dictID = ZSTD_compress_insertDictionary( + &cdict->cBlockState, &cdict->matchState, ¶ms, cdict->dictContent, cdict->dictContentSize, dictContentType, cdict->workspace); if (ZSTD_isError(dictID)) return dictID; @@ -2836,7 +2912,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (U32)pledgedSrcSize); if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; params.fParams.contentSizeFlag = 1; - params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0); + params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, 0); return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); } @@ -2873,9 +2949,6 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, zcs->cdict = cdict; } - params.compressionLevel = ZSTD_CLEVEL_CUSTOM; /* enforce usage of cParams, instead of a dynamic derivation from cLevel (but does that happen ?) */ - zcs->requestedParams = params; - return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); } @@ -2905,20 +2978,22 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); /* note : will check that cdict != NULL */ } + /* ZSTD_initCStream_advanced() : - * pledgedSrcSize must be correct. + * pledgedSrcSize must be exact. * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) { - ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u", (U32)pledgedSrcSize, params.fParams.contentSizeFlag); CHECK_F( ZSTD_checkCParams(params.cParams) ); if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */ - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, cctxParams, pledgedSrcSize); + { ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, cctxParams, pledgedSrcSize); + } } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) @@ -3130,7 +3205,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, DEBUGLOG(4, "ZSTD_compress_generic : transparent init stage"); if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */ params.cParams = ZSTD_getCParamsFromCCtxParams( - cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); + &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); #ifdef ZSTD_MULTITHREAD if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { @@ -3170,7 +3245,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, #ifdef ZSTD_MULTITHREAD if (cctx->appliedParams.nbWorkers > 0) { if (cctx->cParamsChanged) { - ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, cctx->requestedParams.compressionLevel, cctx->requestedParams.cParams); + ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); cctx->cParamsChanged = 0; } { size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); diff --git a/lib/compress/zstd_compress_impl.h b/lib/compress/zstd_compress_impl.h deleted file mode 100644 index e6a30b1d0..000000000 --- a/lib/compress/zstd_compress_impl.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2018-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - -#ifndef FUNCTION -# error "FUNCTION(name) must be defined" -#endif - -#ifndef TARGET -# error "TARGET must be defined" -#endif - - -MEM_STATIC TARGET -size_t FUNCTION(ZSTD_encodeSequences)( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) -{ - BIT_CStream_t blockStream; - FSE_CState_t stateMatchLength; - FSE_CState_t stateOffsetBits; - FSE_CState_t stateLitLength; - - CHECK_E(BIT_initCStream(&blockStream, dst, dstCapacity), dstSize_tooSmall); /* not enough space remaining */ - - /* first symbols */ - FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); - FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); - FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); - BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - if (longOffsets) { - U32 const ofBits = ofCodeTable[nbSeq-1]; - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); - BIT_flushBits(&blockStream); - } - BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, - ofBits - extraBits); - } else { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); - } - BIT_flushBits(&blockStream); - - { size_t n; - for (n=nbSeq-2 ; n = 64-7-(LLFSELog+MLFSELog+OffFSELog))) - BIT_flushBits(&blockStream); /* (7)*/ - BIT_addBits(&blockStream, sequences[n].litLength, llBits); - if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); - if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); - if (longOffsets) { - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[n].offset, extraBits); - BIT_flushBits(&blockStream); /* (7)*/ - } - BIT_addBits(&blockStream, sequences[n].offset >> extraBits, - ofBits - extraBits); /* 31 */ - } else { - BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ - } - BIT_flushBits(&blockStream); /* (7)*/ - } } - - DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); - FSE_flushCState(&blockStream, &stateMatchLength); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); - FSE_flushCState(&blockStream, &stateOffsetBits); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); - FSE_flushCState(&blockStream, &stateLitLength); - - { size_t const streamSize = BIT_closeCStream(&blockStream); - if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ - return streamSize; - } -} diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index c09c2333b..81f12ca6d 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -32,7 +32,6 @@ extern "C" { ***************************************/ #define kSearchStrength 8 #define HASH_READ_SIZE 8 -#define ZSTD_CLEVEL_CUSTOM 999 #define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted". It could be confused for a real successor at index "1", if sorted as larger than its predecessor. It's not a big deal though : candidate will just be sorted again. @@ -183,7 +182,7 @@ struct ZSTD_CCtx_params_s { /* Long distance matching parameters */ ldmParams_t ldmParams; - /* For use with createCCtxParams() and freeCCtxParams() only */ + /* Internal use, for createCCtxParams() and freeCCtxParams() only */ ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ @@ -640,6 +639,13 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, * These prototypes shall only be called from within lib/compress * ============================================================== */ +/* ZSTD_getCParamsFromCCtxParams() : + * cParams are built depending on compressionLevel, src size hints, + * LDM and manually set compression parameters. + */ +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize); + /*! ZSTD_initCStream_internal() : * Private use only. Init streaming operation. * expects params to be valid. diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 5c9c0d2b6..bffd8a3df 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -17,21 +17,14 @@ #define LDM_HASH_RLOG 7 #define LDM_HASH_CHAR_OFFSET 10 -size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm) -{ - ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); - params->enableLdm = enableLdm>0; - params->hashLog = 0; - params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; - params->minMatchLength = LDM_MIN_MATCH_LENGTH; - params->hashEveryLog = ZSTD_LDM_HASHEVERYLOG_NOTSET; - return 0; -} - void ZSTD_ldm_adjustParameters(ldmParams_t* params, ZSTD_compressionParameters const* cParams) { U32 const windowLog = cParams->windowLog; + ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); + DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); + if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; + if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; if (cParams->strategy >= ZSTD_btopt) { /* Get out of the way of the optimal parser */ U32 const minMatch = MAX(cParams->targetLength, params->minMatchLength); @@ -43,7 +36,7 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, params->hashLog = MAX(ZSTD_HASHLOG_MIN, windowLog - LDM_HASH_RLOG); assert(params->hashLog <= ZSTD_HASHLOG_MAX); } - if (params->hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET) { + if (params->hashEveryLog == 0) { params->hashEveryLog = windowLog < params->hashLog ? 0 : windowLog - params->hashLog; } @@ -183,6 +176,7 @@ static U64 ZSTD_ldm_ipow(U64 base, U64 exp) } U64 ZSTD_ldm_getHashPower(U32 minMatchLength) { + DEBUGLOG(4, "ZSTD_ldm_getHashPower: mml=%u", minMatchLength); assert(minMatchLength >= ZSTD_LDM_MINMATCH_MIN); return ZSTD_ldm_ipow(prime8bytes, minMatchLength - 1); } @@ -536,6 +530,34 @@ size_t ZSTD_ldm_generateSequences( return 0; } +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) { + while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + if (srcSize <= seq->litLength) { + /* Skip past srcSize literals */ + seq->litLength -= (U32)srcSize; + return; + } + srcSize -= seq->litLength; + seq->litLength = 0; + if (srcSize < seq->matchLength) { + /* Skip past the first srcSize of the match */ + seq->matchLength -= (U32)srcSize; + if (seq->matchLength < minMatch) { + /* The match is too short, omit it */ + if (rawSeqStore->pos + 1 < rawSeqStore->size) { + seq[1].litLength += seq[0].matchLength; + } + rawSeqStore->pos++; + } + return; + } + srcSize -= seq->matchLength; + seq->matchLength = 0; + rawSeqStore->pos++; + } +} + /** * If the sequence length is longer than remaining then the sequence is split * between this block and the next. @@ -546,51 +568,24 @@ size_t ZSTD_ldm_generateSequences( static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, U32 const remaining, U32 const minMatch) { - size_t const pos = rawSeqStore->pos; rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; assert(sequence.offset > 0); - /* Handle partial sequences */ + /* Likely: No partial sequence */ + if (remaining >= sequence.litLength + sequence.matchLength) { + rawSeqStore->pos++; + return sequence; + } + /* Cut the sequence short (offset == 0 ==> rest is literals). */ if (remaining <= sequence.litLength) { - /* Split the literals that we have out of the sequence. - * They will become the last literals of this block. - * The next block starts off with the remaining literals. - */ - rawSeqStore->seq[pos].litLength -= remaining; sequence.offset = 0; } else if (remaining < sequence.litLength + sequence.matchLength) { - /* Split the match up into two sequences. One in this block, and one - * in the next with no literals. If either match would be shorter - * than searchLength we omit it. - */ - U32 const matchPrefix = remaining - sequence.litLength; - U32 const matchSuffix = sequence.matchLength - matchPrefix; - - assert(remaining > sequence.litLength); - assert(matchPrefix < sequence.matchLength); - assert(matchPrefix + matchSuffix == sequence.matchLength); - /* Update the first sequence */ - sequence.matchLength = matchPrefix; - /* Update the second sequence */ - if (matchSuffix >= minMatch) { - /* Update the second sequence, since the suffix is long enough */ - rawSeqStore->seq[pos].litLength = 0; - rawSeqStore->seq[pos].matchLength = matchSuffix; - } else { - /* Omit the second sequence since the match suffix is too short. - * Add to the next sequences literals (if any). - */ - if (pos + 1 < rawSeqStore->size) - rawSeqStore->seq[pos + 1].litLength += matchSuffix; - rawSeqStore->pos++; /* Consume the sequence */ - } + sequence.matchLength = remaining - sequence.litLength; if (sequence.matchLength < minMatch) { - /* Skip the current sequence if it is too short */ sequence.offset = 0; } - } else { - /* No partial sequence */ - rawSeqStore->pos++; /* Consume the sequence */ } + /* Skip past `remaining` bytes for the future sequences. */ + ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch); return sequence; } diff --git a/lib/compress/zstd_ldm.h b/lib/compress/zstd_ldm.h index 9d2f7c391..0c3789ff1 100644 --- a/lib/compress/zstd_ldm.h +++ b/lib/compress/zstd_ldm.h @@ -22,7 +22,6 @@ extern "C" { ***************************************/ #define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX -#define ZSTD_LDM_HASHEVERYLOG_NOTSET 9999 /** * ZSTD_ldm_generateSequences(): @@ -39,8 +38,8 @@ extern "C" { * sequences. */ size_t ZSTD_ldm_generateSequences( - ldmState_t* ldms, rawSeqStore_t* sequences, - ldmParams_t const* params, void const* src, size_t srcSize); + ldmState_t* ldms, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize); /** * ZSTD_ldm_blockCompress(): @@ -61,14 +60,21 @@ size_t ZSTD_ldm_generateSequences( * NOTE: This function does not return any errors. */ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize, - int const extDict); + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_compressionParameters const* cParams, + void const* src, size_t srcSize, + int const extDict); +/** + * ZSTD_ldm_skipSequences(): + * + * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. + * Avoids emitting matches less than `minMatch` bytes. + * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, + U32 const minMatch); -/** ZSTD_ldm_initializeParameters() : - * Initialize the long distance matching parameters to their default values. */ -size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm); /** ZSTD_ldm_getTableSize() : * Estimate the space needed for long distance matching tables or 0 if LDM is diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 94de683e2..c7b08cbce 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -27,8 +27,14 @@ #include "pool.h" /* threadpool */ #include "threading.h" /* mutex */ #include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ +#include "zstd_ldm.h" #include "zstdmt_compress.h" +/* Guards code to support resizing the SeqPool. + * We will want to resize the SeqPool to save memory in the future. + * Until then, comment the code out since it is unused. + */ +#define ZSTD_RESIZE_SEQPOOL 0 /* ====== Debug ====== */ #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2) @@ -194,6 +200,32 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) } } +#if ZSTD_RESIZE_SEQPOOL +/** ZSTDMT_resizeBuffer() : + * assumption : bufPool must be valid + * @return : a buffer that is at least the buffer pool buffer size. + * If a reallocation happens, the data in the input buffer is copied. + */ +static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) +{ + size_t const bSize = bufPool->bufferSize; + if (buffer.capacity < bSize) { + void* const start = ZSTD_malloc(bSize, bufPool->cMem); + buffer_t newBuffer; + newBuffer.start = start; + newBuffer.capacity = start == NULL ? 0 : bSize; + if (start != NULL) { + assert(newBuffer.capacity >= buffer.capacity); + memcpy(newBuffer.start, buffer.start, buffer.capacity); + DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize); + return newBuffer; + } + DEBUGLOG(5, "ZSTDMT_resizeBuffer: buffer allocation failure !!"); + } + return buffer; +} +#endif + /* store buffer for later re-use, up to pool capacity */ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) { @@ -214,6 +246,72 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) } +/* ===== Seq Pool Wrapper ====== */ + +static rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0}; + +typedef ZSTDMT_bufferPool ZSTDMT_seqPool; + +static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) +{ + return ZSTDMT_sizeof_bufferPool(seqPool); +} + +static rawSeqStore_t bufferToSeq(buffer_t buffer) +{ + rawSeqStore_t seq = {NULL, 0, 0, 0}; + seq.seq = (rawSeq*)buffer.start; + seq.capacity = buffer.capacity / sizeof(rawSeq); + return seq; +} + +static buffer_t seqToBuffer(rawSeqStore_t seq) +{ + buffer_t buffer; + buffer.start = seq.seq; + buffer.capacity = seq.capacity * sizeof(rawSeq); + return buffer; +} + +static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) +{ + if (seqPool->bufferSize == 0) { + return kNullRawSeqStore; + } + return bufferToSeq(ZSTDMT_getBuffer(seqPool)); +} + +#if ZSTD_RESIZE_SEQPOOL +static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq))); +} +#endif + +static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq)); +} + +static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq) +{ + ZSTDMT_setBufferSize(seqPool, nbSeq * sizeof(rawSeq)); +} + +static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem) +{ + ZSTDMT_seqPool* seqPool = ZSTDMT_createBufferPool(nbWorkers, cMem); + ZSTDMT_setNbSeq(seqPool, 0); + return seqPool; +} + +static void ZSTDMT_freeSeqPool(ZSTDMT_seqPool* seqPool) +{ + ZSTDMT_freeBufferPool(seqPool); +} + + + /* ===== CCtx Pool ===== */ /* a single CCtx Pool can be invoked from multiple threads in parallel */ @@ -312,36 +410,97 @@ typedef struct { } range_t; typedef struct { + /* All variables in the struct are protected by mutex. */ ZSTD_pthread_mutex_t mutex; ZSTD_pthread_cond_t cond; ZSTD_CCtx_params params; + ldmState_t ldmState; XXH64_state_t xxhState; unsigned nextJobID; + /* Protects ldmWindow. + * Must be acquired after the main mutex when acquiring both. + */ + ZSTD_pthread_mutex_t ldmWindowMutex; + ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is udpated */ + ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ } serialState_t; -static void ZSTDMT_serialState_reset(serialState_t* serialState, ZSTD_CCtx_params params) +static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool* seqPool, ZSTD_CCtx_params params) { + /* Adjust parameters */ + if (params.ldmParams.enableLdm) { + DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); + params.ldmParams.windowLog = params.cParams.windowLog; + ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashEveryLog < 32); + serialState->ldmState.hashPower = + ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength); + } else { + memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); + } serialState->nextJobID = 0; if (params.fParams.checksumFlag) XXH64_reset(&serialState->xxhState, 0); + if (params.ldmParams.enableLdm) { + ZSTD_customMem cMem = params.customMem; + unsigned const hashLog = params.ldmParams.hashLog; + size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t); + unsigned const bucketLog = + params.ldmParams.hashLog - params.ldmParams.bucketSizeLog; + size_t const bucketSize = (size_t)1 << bucketLog; + unsigned const prevBucketLog = + serialState->params.ldmParams.hashLog - + serialState->params.ldmParams.bucketSizeLog; + /* Size the seq pool tables */ + ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, params.jobSize)); + /* Reset the window */ + ZSTD_window_clear(&serialState->ldmState.window); + serialState->ldmWindow = serialState->ldmState.window; + /* Resize tables and output space if necessary. */ + if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { + ZSTD_free(serialState->ldmState.hashTable, cMem); + serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_malloc(hashSize, cMem); + } + if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) { + ZSTD_free(serialState->ldmState.bucketOffsets, cMem); + serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_malloc(bucketSize, cMem); + } + if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets) + return 1; + /* Zero the tables */ + memset(serialState->ldmState.hashTable, 0, hashSize); + memset(serialState->ldmState.bucketOffsets, 0, bucketSize); + } serialState->params = params; + return 0; } static int ZSTDMT_serialState_init(serialState_t* serialState) { int initError = 0; + memset(serialState, 0, sizeof(*serialState)); initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL); initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL); + initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->ldmWindowCond, NULL); return initError; } static void ZSTDMT_serialState_free(serialState_t* serialState) { + ZSTD_customMem cMem = serialState->params.customMem; ZSTD_pthread_mutex_destroy(&serialState->mutex); ZSTD_pthread_cond_destroy(&serialState->cond); + ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex); + ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond); + ZSTD_free(serialState->ldmState.hashTable, cMem); + ZSTD_free(serialState->ldmState.bucketOffsets, cMem); } -static void ZSTDMT_serialState_update(serialState_t* serialState, range_t src, unsigned jobID) +static void ZSTDMT_serialState_update(serialState_t* serialState, + ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore, + range_t src, unsigned jobID) { /* Wait for our turn */ ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); @@ -351,6 +510,24 @@ static void ZSTDMT_serialState_update(serialState_t* serialState, range_t src, u /* A future job may error and skip our job */ if (serialState->nextJobID == jobID) { /* It is now our turn, do any processing necessary */ + if (serialState->params.ldmParams.enableLdm) { + size_t error; + assert(seqStore.seq != NULL && seqStore.pos == 0 && + seqStore.size == 0 && seqStore.capacity > 0); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size); + error = ZSTD_ldm_generateSequences( + &serialState->ldmState, &seqStore, + &serialState->params.ldmParams, src.start, src.size); + /* We provide a large enough buffer to never fail. */ + assert(!ZSTD_isError(error)); (void)error; + /* Update ldmWindow to match the ldmState.window and signal the main + * thread if it is waiting for a buffer. + */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + serialState->ldmWindow = serialState->ldmState.window; + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } if (serialState->params.fParams.checksumFlag && src.size > 0) XXH64_update(&serialState->xxhState, src.start, src.size); } @@ -358,6 +535,14 @@ static void ZSTDMT_serialState_update(serialState_t* serialState, range_t src, u serialState->nextJobID++; ZSTD_pthread_cond_broadcast(&serialState->cond); ZSTD_pthread_mutex_unlock(&serialState->mutex); + + if (seqStore.size > 0) { + size_t const err = ZSTD_referenceExternalSequences( + jobCCtx, seqStore.seq, seqStore.size); + assert(serialState->params.ldmParams.enableLdm); + assert(!ZSTD_isError(err)); + (void)err; + } } static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, @@ -369,6 +554,11 @@ static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, DEBUGLOG(5, "Skipping past job %u because of error", jobID); serialState->nextJobID = jobID + 1; ZSTD_pthread_cond_broadcast(&serialState->cond); + + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + ZSTD_window_clear(&serialState->ldmWindow); + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); } ZSTD_pthread_mutex_unlock(&serialState->mutex); @@ -388,6 +578,7 @@ typedef struct { ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */ buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ @@ -408,10 +599,15 @@ void ZSTDMT_compressionJob(void* jobDescription) ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); + rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); buffer_t dstBuff = job->dstBuff; - /* Don't compute the checksum for chunks, but write it in the header */ + /* Don't compute the checksum for chunks, since we compute it externally, + * but write it in the header. + */ if (job->jobID != 0) jobParams.fParams.checksumFlag = 0; + /* Don't run LDM for the chunks, since we handle it externally */ + jobParams.ldmParams.enableLdm = 0; /* ressources */ if (cctx==NULL) { @@ -448,8 +644,8 @@ void ZSTDMT_compressionJob(void* jobDescription) goto _endJob; } } } - /* Perform serial step as early as possible */ - ZSTDMT_serialState_update(job->serial, job->src, job->jobID); + /* Perform serial step as early as possible, but after CCtx initialization */ + ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); @@ -504,6 +700,7 @@ _endJob: DEBUGLOG(5, "Finished with prefix: %zx", (size_t)job->prefix.start); DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start); /* release resources */ + ZSTDMT_releaseSeq(job->seqPool, rawSeqStore); ZSTDMT_releaseCCtx(job->cctxPool, cctx); /* report */ ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); @@ -544,6 +741,7 @@ struct ZSTDMT_CCtx_s { ZSTDMT_jobDescription* jobs; ZSTDMT_bufferPool* bufPool; ZSTDMT_CCtxPool* cctxPool; + ZSTDMT_seqPool* seqPool; ZSTD_CCtx_params params; size_t targetSectionSize; size_t targetPrefixSize; @@ -635,9 +833,10 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem) mtctx->jobIDMask = nbJobs - 1; mtctx->bufPool = ZSTDMT_createBufferPool(nbWorkers, cMem); mtctx->cctxPool = ZSTDMT_createCCtxPool(nbWorkers, cMem); + mtctx->seqPool = ZSTDMT_createSeqPool(nbWorkers, cMem); initError = ZSTDMT_serialState_init(&mtctx->serial); mtctx->roundBuff = kNullRoundBuff; - if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool | initError) { + if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool | !mtctx->seqPool | initError) { ZSTDMT_freeCCtx(mtctx); return NULL; } @@ -692,6 +891,7 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); ZSTDMT_freeBufferPool(mtctx->bufPool); ZSTDMT_freeCCtxPool(mtctx->cctxPool); + ZSTDMT_freeSeqPool(mtctx->seqPool); ZSTDMT_serialState_free(&mtctx->serial); ZSTD_freeCDict(mtctx->cdictLocal); if (mtctx->roundBuff.buffer) @@ -708,6 +908,7 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) + ZSTDMT_sizeof_bufferPool(mtctx->bufPool) + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTDMT_sizeof_seqPool(mtctx->seqPool) + ZSTD_sizeof_CDict(mtctx->cdictLocal) + mtctx->roundBuff.capacity; } @@ -761,24 +962,23 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) jobParams.compressionLevel = params.compressionLevel; jobParams.disableLiteralCompression = params.disableLiteralCompression; - jobParams.ldmParams = params.ldmParams; return jobParams; } /*! ZSTDMT_updateCParams_whileCompressing() : - * Update compression level and parameters (except wlog) - * while compression is ongoing. + * Updates only a selected set of compression parameters, to remain compatible with current frame. * New parameters will be applied to next compression job. */ -void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, int compressionLevel, ZSTD_compressionParameters cParams) +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams) { U32 const saved_wlog = mtctx->params.cParams.windowLog; /* Do not modify windowLog while compressing */ + int const compressionLevel = cctxParams->compressionLevel; DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", compressionLevel); mtctx->params.compressionLevel = compressionLevel; - if (compressionLevel != ZSTD_CLEVEL_CUSTOM) - cParams = ZSTD_getCParams(compressionLevel, mtctx->frameContentSize, 0 /* dictSize */ ); - cParams.windowLog = saved_wlog; - mtctx->params.cParams = cParams; + { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, 0, 0); + cParams.windowLog = saved_wlog; + mtctx->params.cParams = cParams; + } } /* ZSTDMT_getNbWorkers(): @@ -827,12 +1027,16 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) static size_t ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) { + if (params.ldmParams.enableLdm) + return MAX(21, params.cParams.chainLog + 4); return params.cParams.windowLog + 2; } static size_t ZSTDMT_computeOverlapLog(ZSTD_CCtx_params const params) { unsigned const overlapRLog = (params.overlapSizeLog>9) ? 0 : 9-params.overlapSizeLog; + if (params.ldmParams.enableLdm) + return (MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog); return overlapRLog >= 9 ? 0 : (params.cParams.windowLog - overlapRLog); } @@ -856,7 +1060,7 @@ static size_t ZSTDMT_compress_advanced_internal( void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, - ZSTD_CCtx_params const params) + ZSTD_CCtx_params params) { ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(params); size_t const overlapSize = (size_t)1 << ZSTDMT_computeOverlapLog(params); @@ -870,6 +1074,7 @@ static size_t ZSTDMT_compress_advanced_internal( assert(jobParams.nbWorkers == 0); assert(mtctx->cctxPool->totalCCtx == params.nbWorkers); + params.jobSize = (U32)avgJobSize; DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: nbJobs=%2u (rawSize=%u bytes; fixedSize=%u) ", nbJobs, (U32)proposedJobSize, (U32)avgJobSize); @@ -882,7 +1087,8 @@ static size_t ZSTDMT_compress_advanced_internal( assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */ ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgJobSize) ); - ZSTDMT_serialState_reset(&mtctx->serial, params); + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params)) + return ERROR(memory_allocation); if (nbJobs > mtctx->jobIDMask+1) { /* enlarge job table */ U32 jobsTableSize = nbJobs; @@ -915,6 +1121,7 @@ static size_t ZSTDMT_compress_advanced_internal( mtctx->jobs[u].dstBuff = dstBuffer; mtctx->jobs[u].cctxPool = mtctx->cctxPool; mtctx->jobs[u].bufPool = mtctx->bufPool; + mtctx->jobs[u].seqPool = mtctx->seqPool; mtctx->jobs[u].serial = &mtctx->serial; mtctx->jobs[u].jobID = u; mtctx->jobs[u].firstJob = (u==0); @@ -1023,10 +1230,7 @@ size_t ZSTDMT_initCStream_internal( /* init */ if (params.jobSize == 0) { - if (params.cParams.windowLog >= 29) - params.jobSize = ZSTDMT_JOBSIZE_MAX; - else - params.jobSize = 1U << ZSTDMT_computeTargetJobLog(params); + params.jobSize = 1U << ZSTDMT_computeTargetJobLog(params); } if (params.jobSize > ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX; @@ -1072,11 +1276,20 @@ size_t ZSTDMT_initCStream_internal( DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); { - /* Two buffers of slack, plus extra space for the overlap */ + /* If ldm is enabled we need windowSize space. */ + size_t const windowSize = mtctx->params.ldmParams.enableLdm ? (1U << mtctx->params.cParams.windowLog) : 0; + /* Two buffers of slack, plus extra space for the overlap + * This is the minimum slack that LDM works with. One extra because + * flush might waste up to targetSectionSize-1 bytes. Another extra + * for the overlap (if > 0), then one to fill which doesn't overlap + * with the LDM window. + */ + size_t const nbSlackBuffers = 2 + (mtctx->targetPrefixSize > 0); + size_t const slackSize = mtctx->targetSectionSize * nbSlackBuffers; + /* Compute the total size, and always have enough slack */ size_t const nbWorkers = MAX(mtctx->params.nbWorkers, 1); - size_t const nbSlackBuffers = MIN(nbWorkers, 2) + (mtctx->targetPrefixSize > 0); - size_t const nbSections = nbWorkers + nbSlackBuffers; - size_t const capacity = mtctx->targetSectionSize * nbSections; + size_t const sectionsSize = mtctx->targetSectionSize * nbWorkers; + size_t const capacity = MAX(windowSize, sectionsSize) + slackSize; if (mtctx->roundBuff.capacity < capacity) { if (mtctx->roundBuff.buffer) ZSTD_free(mtctx->roundBuff.buffer, mtctx->cMem); @@ -1099,7 +1312,8 @@ size_t ZSTDMT_initCStream_internal( mtctx->allJobsCompleted = 0; mtctx->consumed = 0; mtctx->produced = 0; - ZSTDMT_serialState_reset(&mtctx->serial, params); + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params)) + return ERROR(memory_allocation); return 0; } @@ -1201,6 +1415,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS mtctx->jobs[jobID].dstBuff = g_nullBuffer; mtctx->jobs[jobID].cctxPool = mtctx->cctxPool; mtctx->jobs[jobID].bufPool = mtctx->bufPool; + mtctx->jobs[jobID].seqPool = mtctx->seqPool; mtctx->jobs[jobID].serial = &mtctx->serial; mtctx->jobs[jobID].jobID = mtctx->nextJobID; mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0); @@ -1383,10 +1598,51 @@ static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range) if (rangeStart == NULL || bufferStart == NULL) return 0; + /* Empty ranges cannot overlap */ + if (bufferStart == bufferEnd || rangeStart == rangeEnd) + return 0; return bufferStart < rangeEnd && rangeStart < bufferEnd; } +static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) +{ + range_t extDict; + range_t prefix; + + extDict.start = window.dictBase + window.lowLimit; + extDict.size = window.dictLimit - window.lowLimit; + + prefix.start = window.base + window.dictLimit; + prefix.size = window.nextSrc - (window.base + window.dictLimit); + DEBUGLOG(5, "extDict [0x%zx, 0x%zx)", + (size_t)extDict.start, + (size_t)extDict.start + extDict.size); + DEBUGLOG(5, "prefix [0x%zx, 0x%zx)", + (size_t)prefix.start, + (size_t)prefix.start + prefix.size); + + return ZSTDMT_isOverlapped(buffer, extDict) + || ZSTDMT_isOverlapped(buffer, prefix); +} + +static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) +{ + if (mtctx->params.ldmParams.enableLdm) { + ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; + DEBUGLOG(5, "source [0x%zx, 0x%zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + ZSTD_PTHREAD_MUTEX_LOCK(mutex); + while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) { + DEBUGLOG(6, "Waiting for LDM to finish..."); + ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex); + } + DEBUGLOG(6, "Done waiting for LDM to finish"); + ZSTD_pthread_mutex_unlock(mutex); + } +} + /** * Attempts to set the inBuff to the next section to fill. * If any part of the new section is still in use we give up. @@ -1415,6 +1671,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) DEBUGLOG(6, "Waiting for buffer..."); return 0; } + ZSTDMT_waitForLdmComplete(mtctx, buffer); memmove(start, mtctx->inBuff.prefix.start, prefixSize); mtctx->inBuff.prefix.start = start; mtctx->roundBuff.pos = prefixSize; @@ -1427,6 +1684,9 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) return 0; } assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix)); + + ZSTDMT_waitForLdmComplete(mtctx, buffer); + DEBUGLOG(5, "Using prefix range [%zx, %zx)", (size_t)mtctx->inBuff.prefix.start, (size_t)mtctx->inBuff.prefix.start + mtctx->inBuff.prefix.size); diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 7adaa803f..f79e3b441 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -122,10 +122,9 @@ size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_param size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers); /*! ZSTDMT_updateCParams_whileCompressing() : - * Update compression level and parameters (except wlog) - * while compression is ongoing. + * Updates only a selected set of compression parameters, to remain compatible with current frame. * New parameters will be applied to next compression job. */ -void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, int compressionLevel, ZSTD_compressionParameters cParams); +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams); /* ZSTDMT_getNbWorkers(): * @return nb threads currently active in mtctx. diff --git a/lib/zstd.h b/lib/zstd.h index 5d1466e76..a35091a58 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -907,9 +907,8 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); * and then applied on all subsequent compression jobs. * When no parameter is ever provided, CCtx is created with compression level ZSTD_CLEVEL_DEFAULT. * - * This API is intended to replace all others experimental API. - * It can basically do all other use cases, and even new ones. - * It stands a reasonable chance to become "stable", after a good testing period. + * This API is intended to replace all others advanced / experimental API entry points. + * But it stands a reasonable chance to become "stable", after a reasonable testing period. */ /* note on naming convention : @@ -926,12 +925,12 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); * All enum will be pinned to explicit values before reaching "stable API" status */ typedef enum { - /* Question : should we have a format ZSTD_f_auto ? - * For the time being, it would mean exactly the same as ZSTD_f_zstd1. - * But, in the future, should several formats be supported, + /* Opened question : should we have a format ZSTD_f_auto ? + * Today, it would mean exactly the same as ZSTD_f_zstd1. + * But, in the future, should several formats become supported, * on the compression side, it would mean "default format". - * On the decompression side, it would mean "multi format", - * and ZSTD_f_zstd1 could be reserved to mean "accept *only* zstd frames". + * On the decompression side, it would mean "automatic format detection", + * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames". * Since meaning is a little different, another option could be to define different enums for compression and decompression. * This question could be kept for later, when there are actually multiple formats to support, * but there is also the question of pinning enum values, and pinning value `0` is especially important */ @@ -951,33 +950,34 @@ typedef enum { * Default level is ZSTD_CLEVEL_DEFAULT==3. * Special: value 0 means "do not change cLevel". * Note 1 : it's possible to pass a negative compression level by casting it to unsigned type. - * Note 2 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */ + * Note 2 : setting a level sets all default values of other compression parameters. + * Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */ ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2. * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. - * Special: value 0 means "do not change windowLog". + * Special: value 0 means "use default windowLog". * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27) - * requires setting the maximum window size at least as large during decompression. */ + * requires explicitly allowing such window size during decompression stage. */ ZSTD_p_hashLog, /* Size of the probe table, as a power of 2. * Resulting table size is (1 << (hashLog+2)). * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. * Larger tables improve compression ratio of strategies <= dFast, * and improve speed of strategies > dFast. - * Special: value 0 means "do not change hashLog". */ + * Special: value 0 means "use default hashLog". */ ZSTD_p_chainLog, /* Size of the full-search table, as a power of 2. * Resulting table size is (1 << (chainLog+2)). * Larger tables result in better and slower compression. * This parameter is useless when using "fast" strategy. - * Special: value 0 means "do not change chainLog". */ + * Special: value 0 means "use default chainLog". */ ZSTD_p_searchLog, /* Number of search attempts, as a power of 2. * More attempts result in better and slower compression. * This parameter is useless when using "fast" and "dFast" strategies. - * Special: value 0 means "do not change searchLog". */ + * Special: value 0 means "use default searchLog". */ ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller). * Larger values make faster compression and decompression, but decrease ratio. * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX. * Note that currently, for all strategies < btopt, effective minimum is 4. - * Note that currently, for all strategies > fast, effective maximum is 6. - * Special: value 0 means "do not change minMatchLength". */ + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ ZSTD_p_targetLength, /* Impact of this field depends on strategy. * For strategies btopt & btultra: * Length of Match considered "good enough" to stop search. @@ -985,12 +985,39 @@ typedef enum { * For strategy fast: * Distance between match sampling. * Larger values make compression faster, and weaker. - * Special: value 0 means "do not change targetLength". */ + * Special: value 0 means "use default targetLength". */ ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition. * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility. * The higher the value of selected strategy, the more complex it is, * resulting in stronger and slower compression. - * Special: value 0 means "do not change strategy". */ + * Special: value 0 means "use default strategy". */ + + ZSTD_p_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases ZSTD_p_windowLog to 128 MB + * except when expressly set to a different value. */ + ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_p_ldmMinMatch, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX . + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashEveryLog". */ /* frame parameters */ ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) @@ -1017,7 +1044,9 @@ typedef enum { ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job. * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */ - /* advanced parameters - may not remain available after API update */ + /* =================================================================== */ + /* experimental parameters - no stability guaranteed */ + /* =================================================================== */ ZSTD_p_compressLiterals=1000, /* control huffman compression of literals (enabled) by default. * disabling it improves speed and decreases compression ratio by a large amount. @@ -1028,40 +1057,6 @@ typedef enum { ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, * even when referencing into Dictionary content (default:0) */ - ZSTD_p_enableLongDistanceMatching=1200, /* Enable long distance matching. - * This parameter is designed to improve the compression - * ratio for large inputs with long distance matches. - * This increases the memory usage as well as window size. - * Note: setting this parameter sets all the LDM parameters - * as well as ZSTD_p_windowLog. It should be set after - * ZSTD_p_compressionLevel and before ZSTD_p_windowLog and - * other LDM parameters. Setting the compression level - * after this parameter overrides the window log, though LDM - * will remain enabled until explicitly disabled. */ - ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2. - * Larger values increase memory usage and compression ratio, but decrease - * compression speed. - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX - * (default: windowlog - 7). - * Special: value 0 means "do not change ldmHashLog". */ - ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher. - * Larger/too small values usually decrease compression ratio. - * Must be clamped between ZSTD_LDM_MINMATCH_MIN - * and ZSTD_LDM_MINMATCH_MAX (default: 64). - * Special: value 0 means "do not change ldmMinMatch". */ - ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution. - * Larger values usually improve collision resolution but may decrease - * compression speed. - * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX (default: 3). - * note : 0 is a valid value */ - ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table. - * The default is MAX(0, (windowLog - ldmHashLog)) to - * optimize hash table usage. - * Larger values improve compression speed. Deviating far from the - * default value will likely result in a decrease in compression ratio. - * Must be clamped between 0 and ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN. - * note : 0 is a valid value */ - } ZSTD_cParameter; @@ -1134,6 +1129,16 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); +/*! ZSTD_CCtx_reset() : + * Return a CCtx to clean state. + * Useful after an error, or to interrupt an ongoing compression job and start a new one. + * Any internal data not yet flushed is cancelled. + * Dictionary (if any) is dropped. + * All parameters are back to default values. + * It's possible to modify compression parameters after a reset. + */ +ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); + typedef enum { @@ -1168,16 +1173,6 @@ ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, ZSTD_inBuffer* input, ZSTD_EndDirective endOp); -/*! ZSTD_CCtx_reset() : - * Return a CCtx to clean state. - * Useful after an error, or to interrupt an ongoing compression job and start a new one. - * Any internal data not yet flushed is cancelled. - * Dictionary (if any) is dropped. - * All parameters are back to default values. - * It's possible to modify compression parameters after a reset. - */ -ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); - /*! ZSTD_compress_generic_simpleArgs() : * Same as ZSTD_compress_generic(), @@ -1211,25 +1206,26 @@ ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs ( * for static allocation for single-threaded compression. */ ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); -/*! ZSTD_resetCCtxParams() : + +/*! ZSTD_CCtxParams_reset() : * Reset params to default values. */ -ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params); +ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); -/*! ZSTD_initCCtxParams() : +/*! ZSTD_CCtxParams_init() : * Initializes the compression parameters of cctxParams according to * compression level. All other parameters are reset to their default values. */ -ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, int compressionLevel); +ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); -/*! ZSTD_initCCtxParams_advanced() : +/*! ZSTD_CCtxParams_init_advanced() : * Initializes the compression and frame parameters of cctxParams according to * params. All other parameters are reset to their default values. */ -ZSTDLIB_API size_t ZSTD_initCCtxParams_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); +ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); -ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /*! ZSTD_CCtxParam_setParameter() : * Similar to ZSTD_CCtx_setParameter. diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 82b686052..e97b841e8 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1024,6 +1024,45 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeCCtx(cctx); } + /* parameters order test */ + { size_t const inputSize = CNBuffSize / 2; + U64 xxh64; + + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) ); + { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; + ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; + size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + if (result != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + cSize = out.pos; + xxh64 = XXH64(out.dst, out.pos, 0); + } + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize); + ZSTD_freeCCtx(cctx); + } + + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_enableLongDistanceMatching, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) ); + { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; + ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; + size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end); + if (result != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + if (out.pos != cSize) goto _output_error; /* must result in same compressed result, hence same size */ + if (XXH64(out.dst, out.pos, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)out.pos); + } + ZSTD_freeCCtx(cctx); + } + } + /* custom formats tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */