lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-08 17:22:10 +03:00
Code Activity

added altering dictionary size depending on compression level

Browse Source
This commit is contained in:
Paul Cruz
2017-07-13 10:15:27 -07:00
parent 7c886db0a8
commit 766663f1f1
1 changed files with 15 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
contrib/adaptive-compression/adapt.c
View File

@@ -84,7 +84,6 @@ typedef struct {
pthread_mutex_t jobWrite_mutex; pthread_mutex_t jobWrite_mutex;
pthread_cond_t jobWrite_cond; pthread_cond_t jobWrite_cond;
size_t lastDictSize; size_t lastDictSize;
size_t targetDictSize;
inBuff_t input; inBuff_t input;
cStat_t stats; cStat_t stats;
jobDescription* jobs; jobDescription* jobs;
@@ -148,7 +147,6 @@ static adaptCCtx* createCCtx(unsigned numJobs, const char* const outFilename)
ctx->jobReadyID = 0; ctx->jobReadyID = 0;
ctx->jobCompressedID = 0; ctx->jobCompressedID = 0;
ctx->jobWriteID = 0; ctx->jobWriteID = 0;
ctx->targetDictSize = 1 << 12;
ctx->lastDictSize = 0; ctx->lastDictSize = 0;
ctx->jobs = calloc(1, numJobs*sizeof(jobDescription)); ctx->jobs = calloc(1, numJobs*sizeof(jobDescription));
/* initializing jobs */ /* initializing jobs */
@@ -255,6 +253,14 @@ static unsigned adaptCompressionLevel(adaptCCtx* ctx)
return ctx->compressionLevel; return ctx->compressionLevel;
} }
static size_t getUseableDictSize(unsigned compressionLevel)
{
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0);
unsigned overlapLog = compressionLevel >= (unsigned)ZSTD_maxCLevel() ? 0 : 3;
size_t overlapSize = 1 << (params.cParams.windowLog - overlapLog);
return overlapSize;
}
static void* compressionThread(void* arg) static void* compressionThread(void* arg)
{ {
adaptCCtx* ctx = (adaptCCtx*)arg; adaptCCtx* ctx = (adaptCCtx*)arg;
@@ -281,8 +287,10 @@ static void* compressionThread(void* arg)
DEBUG(3, "compression level used: %u\n", cLevel); DEBUG(3, "compression level used: %u\n", cLevel);
/* begin compression */ /* begin compression */
{ {
size_t useDictSize = MIN(getUseableDictSize(cLevel), job->dictSize);
DEBUG(2, "useDictSize: %zu, job->dictSize: %zu\n", useDictSize, job->dictSize);
size_t const dictModeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceRawDict, 1); size_t const dictModeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceRawDict, 1);
size_t const initError = ZSTD_compressBegin_usingDict(ctx->cctx, job->src.start, job->dictSize, cLevel); size_t const initError = ZSTD_compressBegin_usingDict(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, cLevel);
size_t const windowSizeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceWindow, 1); size_t const windowSizeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceWindow, 1);
if (ZSTD_isError(dictModeError) || ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) { if (ZSTD_isError(dictModeError) || ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) {
DISPLAY("Error: something went wrong while starting compression\n"); DISPLAY("Error: something went wrong while starting compression\n");
@@ -435,12 +443,11 @@ static int createCompressionJob(adaptCCtx* ctx, size_t srcSize, int last)
DEBUG(3, "filled: %zu, srcSize: %zu\n", ctx->input.filled, srcSize); DEBUG(3, "filled: %zu, srcSize: %zu\n", ctx->input.filled, srcSize);
/* if not on the last job, reuse data as dictionary in next job */ /* if not on the last job, reuse data as dictionary in next job */
if (!last) { if (!last) {
size_t const newDictSize = ctx->targetDictSize;
size_t const oldDictSize = ctx->lastDictSize; size_t const oldDictSize = ctx->lastDictSize;
DEBUG(3, "newDictSize %zu oldDictSize %zu\n", newDictSize, oldDictSize); DEBUG(3, "oldDictSize %zu\n", oldDictSize);
memmove(ctx->input.buffer.start, ctx->input.buffer.start + oldDictSize + srcSize - newDictSize, newDictSize); memmove(ctx->input.buffer.start, ctx->input.buffer.start + oldDictSize, srcSize);
ctx->lastDictSize = newDictSize; ctx->lastDictSize = srcSize;
ctx->input.filled = newDictSize; ctx->input.filled = srcSize;
} }
return 0; return 0;
} }