From f12c1304a80a1b8ba53a3352896ab097b6ea589c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 5 Nov 2015 18:16:59 +0100 Subject: [PATCH] integrate fast mode --- Makefile | 2 +- NEWS | 3 +++ lib/zstd.h | 2 +- lib/zstdhc.c | 6 +++++- lib/zstdhc_static.h | 11 +++++++---- programs/Makefile | 2 +- programs/paramgrill.c | 40 ++++++++++++++++++++++------------------ programs/zstdcli.c | 2 +- 8 files changed, 41 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 34d425e09..7c3fab5ab 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ # ################################################################ # Version number -export VERSION := 0.3.3 +export VERSION := 0.3.4 PRGDIR = programs ZSTDDIR = lib diff --git a/NEWS b/NEWS index ee8c47bea..c0cf789eb 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v0.3.4 +Faster low cLevels + v0.3.3 Small compression ratio improvement diff --git a/lib/zstd.h b/lib/zstd.h index d79410d15..115046df8 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -48,7 +48,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MINOR 3 /* for new (non-breaking) interface capabilities */ -#define ZSTD_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ +#define ZSTD_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) unsigned ZSTD_versionNumber (void); diff --git a/lib/zstdhc.c b/lib/zstdhc.c index fa059cb97..aec69d33e 100644 --- a/lib/zstdhc.c +++ b/lib/zstdhc.c @@ -225,10 +225,12 @@ size_t ZSTD_HC_compressBlock_fast_generic(ZSTD_HC_CCtx* ctx, const U32 hBits = ctx->params.hashLog; seqStore_t* seqStorePtr = &(ctx->seqStore); const BYTE* const base = ctx->base; + const size_t maxDist = ((size_t)1 << ctx->params.windowLog); const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart + 1; const BYTE* anchor = istart; + const BYTE* const lowest = (size_t)(istart-base) > maxDist ? istart-maxDist : base; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; @@ -253,7 +255,9 @@ size_t ZSTD_HC_compressBlock_fast_generic(ZSTD_HC_CCtx* ctx, hashTable[h] = (U32)(ip-base); if (MEM_read32(ip-offset_2) == MEM_read32(ip)) match = ip-offset_2; - if (MEM_read32(match) != MEM_read32(ip)) { ip += ((ip-anchor) >> g_searchStrength) + 1; offset_2 = offset_1; continue; } + if ( (match < lowest) || + (MEM_read32(match) != MEM_read32(ip)) ) + { ip += ((ip-anchor) >> g_searchStrength) + 1; offset_2 = offset_1; continue; } while ((ip>anchor) && (match>base) && (ip[-1] == match[-1])) { ip--; match--; } /* catch up */ { diff --git a/lib/zstdhc_static.h b/lib/zstdhc_static.h index b1bf5af82..b0cf10523 100644 --- a/lib/zstdhc_static.h +++ b/lib/zstdhc_static.h @@ -51,7 +51,7 @@ typedef enum { ZSTD_HC_fast, ZSTD_HC_greedy, ZSTD_HC_lazy, ZSTD_HC_lazy2, ZSTD_H typedef struct { U32 windowLog; /* largest match distance : impact decompression buffer size */ - U32 contentLog; /* full search segment : larger == more compression, slower, more memory*/ + U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */ U32 hashLog; /* dispatch table : larger == more memory, faster*/ U32 searchLog; /* nb of searches : larger == more compression, slower*/ U32 searchLength; /* size of matches : larger == faster decompression */ @@ -98,11 +98,11 @@ size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize); /* ************************************* * Pre-defined compression levels ***************************************/ -#define ZSTD_HC_MAX_CLEVEL 22 +#define ZSTD_HC_MAX_CLEVEL 25 static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = { /* W, C, H, S, L, strat */ - { 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 0 - never used */ - { 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 1 - in fact redirected towards zstd fast */ + { 18, 12, 14, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */ + { 18, 12, 14, 1, 7, ZSTD_HC_fast }, /* level 1 - in fact redirected towards zstd fast */ { 18, 12, 15, 2, 4, ZSTD_HC_greedy }, /* level 2 */ { 19, 14, 18, 2, 5, ZSTD_HC_greedy }, /* level 3 */ { 20, 17, 19, 3, 5, ZSTD_HC_greedy }, /* level 4 */ @@ -124,6 +124,9 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] { 25, 25, 23, 6, 5, ZSTD_HC_btlazy2 }, /* level 20 */ { 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 21 */ { 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 22 */ + { 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 23 */ + { 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 24 */ + { 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 25 */ }; diff --git a/programs/Makefile b/programs/Makefile index 8c7003232..c3a250e57 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,7 +30,7 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -VERSION?= 0.3.3 +VERSION?= 0.3.4 DESTDIR?= PREFIX ?= /usr/local diff --git a/programs/paramgrill.c b/programs/paramgrill.c index db0ca790a..a188a41c5 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -487,7 +487,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params, BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); - for (cLevel = 2; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++) + for (cLevel = 1; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++) { if (testResult.cSpeed < g_cSpeedTarget[cLevel]) continue; /* not fast enough for this level */ @@ -513,8 +513,10 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params, double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) + 4 * (1 << params.contentLog); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) + 4 * (1 << winners[cLevel].params.contentLog); + size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) + + ((params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << params.contentLog)); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) + + ((winners[cLevel].params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << winners[cLevel].params.contentLog)); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); @@ -669,8 +671,8 @@ static void BMK_selectRandomStart( const void* srcBuffer, size_t srcSize, ZSTD_HC_CCtx* ctx) { - U32 id = (FUZ_rand(&g_rand) % ZSTD_HC_MAX_CLEVEL) + 1; - if ((id<2) || (winners[id].params.windowLog==0)) + U32 id = (FUZ_rand(&g_rand) % (ZSTD_HC_MAX_CLEVEL+1)); + if ((id==0) || (winners[id].params.windowLog==0)) { /* totally random entry */ ZSTD_HC_parameters p; @@ -712,27 +714,29 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (g_target) - g_cSpeedTarget[2] = g_target * 1000; + g_cSpeedTarget[1] = g_target * 1000; else { - /* baseline config for level 2 */ + /* baseline config for level 1 */ BMK_result_t testResult; - params = g_seedParams[2]; - params.windowLog = MIN(srcLog, params.windowLog); - params.contentLog = MIN(params.windowLog, params.contentLog); - params.searchLog = MIN(params.contentLog, params.searchLog); + params.windowLog = MIN(srcLog, 18); + params.hashLog = 14; + params.contentLog = 1; + params.searchLog = 1; + params.searchLength = 7; + params.strategy = ZSTD_HC_fast; BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); - g_cSpeedTarget[2] = (testResult.cSpeed * 15) >> 4; + g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4; } - /* establish speed objectives (relative to level 2) */ - for (i=3; i<=ZSTD_HC_MAX_CLEVEL; i++) - g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 13) >> 4; + /* establish speed objectives (relative to level 1) */ + for (i=2; i<=ZSTD_HC_MAX_CLEVEL; i++) + g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) >> 5; /* populate initial solution */ { - const int maxSeeds = g_noSeed ? 2 : ZSTD_HC_MAX_CLEVEL; - for (i=2; i<=maxSeeds; i++) + const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL; + for (i=1; i<=maxSeeds; i++) { params = g_seedParams[i]; params.windowLog = MIN(srcLog, params.windowLog); @@ -1004,7 +1008,7 @@ int main(int argc, char** argv) } break; - /* target level2 speed objective, in MB/s */ + /* target level1 speed objective, in MB/s */ case 'T': argument++; g_target = 0; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 4c8460d8a..2cb59c76b 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -70,7 +70,7 @@ **************************************/ #define COMPRESSOR_NAME "zstd command line interface" #ifndef ZSTD_VERSION -# define ZSTD_VERSION "v0.3.3" +# define ZSTD_VERSION "v0.3.4" #endif #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__