From 60fa90b6c04c69cbd9312155f70dad46f2ce92b5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 1 Feb 2018 16:13:31 -0800 Subject: [PATCH] zstdmt: added ability to change compression parameters during compression --- lib/compress/zstd_compress.c | 7 +++++++ lib/compress/zstdmt_compress.c | 21 ++++++++++++++++++++- lib/compress/zstdmt_compress.h | 27 +++++++++++++++++++++------ lib/zstd.h | 2 +- 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4a825f65f..18da77f8d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -476,6 +476,9 @@ size_t ZSTD_CCtxParam_setParameter( /** ZSTD_CCtx_setParametersUsingCCtxParams() : * just applies `params` into `cctx` * no action is performed, parameters are merely stored. + * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. + * This is possible even if a compression is ongoing. + * In which case, new parameters will be applied on the fly, starting with next compression job. */ size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) @@ -484,6 +487,10 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( if (cctx->cdict) return ERROR(stage_wrong); cctx->requestedParams = *params; +#ifdef ZSTD_MULTITHREAD + if (cctx->mtctx) + ZSTDMT_MTCtx_setParametersUsingCCtxParams(cctx->mtctx, params); +#endif return 0; } diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index ec5f0bbbd..32390a61c 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -664,6 +664,25 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) return jobParams; } +/*! ZSTDMT_MTCtx_setParametersUsingCCtxParams() : + * Apply a ZSTD_CCtx_params to the compression context. + * This entry point is accessed while compression is ongoing, + * new parameters will be applied to next compression job. + * However, following parameters are NOT updated : + * - window size + * - pledgedSrcSize + * - nb threads + * - job size + * - overlap size + */ +void ZSTDMT_MTCtx_setParametersUsingCCtxParams(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* params) +{ + U32 const wlog = mtctx->params.cParams.windowLog; + mtctx->params = *params; + mtctx->params.cParams.windowLog = wlog; /* Do not modify windowLog ! */ + /* note : other parameters not updated are simply not used beyond initialization */ +} + /* ZSTDMT_getNbThreads(): * @return nb threads currently active in mtctx. * mtctx must be valid */ @@ -856,7 +875,7 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, - ZSTD_parameters const params, + ZSTD_parameters params, unsigned overlapLog) { ZSTD_CCtx_params cctxParams = mtctx->params; diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 34dfc488f..dcff1138a 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -38,7 +38,7 @@ ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); -/* === Simple buffer-to-butter one-pass function === */ +/* === Simple one-pass compression function === */ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, @@ -50,7 +50,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, /* === Streaming functions === */ ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); -ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it may change in the future, to mean "empty" */ +ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */ ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); @@ -68,7 +68,7 @@ ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, - ZSTD_parameters const params, + ZSTD_parameters params, unsigned overlapLog); ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, @@ -109,15 +109,30 @@ ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, ZSTD_EndDirective endOp); -/* === Private definitions; never ever use directly === */ +/* ======================================================== + * === Private interface, for use by ZSTD_compress.c === + * === Not exposed in libzstd. Never invoke directly === + * ======================================================== */ size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value); /* ZSTDMT_CCtxParam_setNbThreads() - * Set nbThreads, and clamp it correctly, - * also reset jobSize and overlapLog */ + * Set nbThreads, and clamp it. + * Also reset jobSize and overlapLog */ size_t ZSTDMT_CCtxParam_setNbThreads(ZSTD_CCtx_params* params, unsigned nbThreads); +/*! ZSTDMT_MTCtx_setParametersUsingCCtxParams() : + * Apply a ZSTD_CCtx_params to the compression context. + * This works even during compression, and will be applied to next compression job. + * However, the following parameters will NOT be updated after compression has been started : + * - window size + * - pledgedSrcSize + * - nb threads + * - job size + * - overlap size + */ +void ZSTDMT_MTCtx_setParametersUsingCCtxParams(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* params); + /* ZSTDMT_getNbThreads(): * @return nb threads currently active in mtctx. * mtctx must be valid */ diff --git a/lib/zstd.h b/lib/zstd.h index 29c3ced18..be73d25b4 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1198,7 +1198,7 @@ ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs ( ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); /*! ZSTD_resetCCtxParams() : - * Reset params to default, with the default compression level. + * Reset params to default values. */ ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);