From 04b12d8bcb009b522d9ce58bc1e96f11a386a5f3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 11 Feb 2016 06:23:24 +0100 Subject: [PATCH] paramgrill can select manually targetLength new compression levels --- lib/zstd_compress.c | 26 +++++++++++++------------- lib/zstd_opt.h | 16 ++++++---------- programs/paramgrill.c | 22 +++++++++++++--------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index dda6683e6..3ffacc81e 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -903,7 +903,7 @@ static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLim return (size_t)(pIn - pStart); } -/** ZSTD_count_2segments +/** ZSTD_count_2segments() : * can count match length with ip & match in potentially 2 different segments. * convention : on reaching mEnd, match count continue starting from iStart */ @@ -1190,11 +1190,11 @@ void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, } -/* ************************************* +/*-************************************* * Binary Tree search ***************************************/ -/** ZSTD_insertBt1() : add one or multiple positions to tree -* ip : assumed <= iend-8 +/** ZSTD_insertBt1() : add one or multiple positions to tree. +* ip : assumed <= iend-8 . * @return : nb of positions added */ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares, U32 extDict) @@ -1230,7 +1230,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co while (nbCompares-- && (matchIndex > windowLow)) { U32* nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - +#if 1 /* note : can create issues when hlog small <= 11 */ const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ if (matchIndex == predictedSmall) { /* no need to check length, result known */ @@ -1249,7 +1249,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co predictedLarge = predictPtr[0] + (predictPtr[0]>0); continue; } - +#endif if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { match = base + matchIndex; if (match[matchLength] == ip[matchLength]) @@ -1284,7 +1284,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co } } *smallerPtr = *largerPtr = 0; - return (matchEndIdx > current + 8) ? matchEndIdx - current - 8 : 1; + return (matchEndIdx > current + 8) ? (matchEndIdx - current) - 8 : 1; } @@ -2339,12 +2339,12 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { 0, 0, 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */ { 0, 0, 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */ { 0, 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */ - { 0, 0, 26, 27, 25, 9, 5, ZSTD_btlazy2 }, /* level 20 */ - { 0, 0, 23, 21, 22, 5, 4, ZSTD_btlazy2 }, /* level 21 = 16 + L=4 */ // 41233150 btlazy1=41560211 norep1=42322286 - { 0, 12, 23, 21, 22, 5, 4, ZSTD_opt }, /* level 22 */ - { 0, 32, 23, 21, 22, 5, 4, ZSTD_opt }, /* level 23 */ - { 0, 32, 23, 21, 22, 5, 4, ZSTD_opt_bt }, /* level 24 = 16 + btopt */ - { 0, 64, 26, 27, 25, 10, 4, ZSTD_opt_bt }, /* level 25 = 20 + btopt */ + { 0, 12, 22, 20, 21, 3, 5, ZSTD_opt }, /* level 20 */ + { 0, 16, 23, 21, 22, 4, 4, ZSTD_opt }, /* level 21 */ + { 0, 32, 25, 25, 24, 5, 4, ZSTD_opt_bt }, /* level 22 */ + { 0, 64, 25, 26, 24, 6, 4, ZSTD_opt_bt }, /* level 23 */ + { 0,128, 26, 26, 25, 8, 4, ZSTD_opt_bt }, /* level 24 */ + { 0,256, 26, 27, 25, 10, 4, ZSTD_opt_bt }, /* level 25 */ }, { /* for srcSize <= 256 KB */ /* SL, W, C, H, S, L, strat */ diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index a936d3131..b968bdb0a 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -269,7 +269,7 @@ U32 ZSTD_BtGetAllMatches ( } -FORCE_INLINE U32 ZSTD_BtGetAllMatches_selectMLS ( +static U32 ZSTD_BtGetAllMatches_selectMLS ( ZSTD_CCtx* zc, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml) @@ -297,7 +297,7 @@ U32 ZSTD_BtGetAllMatches_extDict ( } -FORCE_INLINE U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( +static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( ZSTD_CCtx* zc, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml) @@ -382,7 +382,7 @@ U32 ZSTD_HcGetAllMatches_generic ( } -FORCE_INLINE U32 ZSTD_HcGetAllMatches_selectMLS ( +static U32 ZSTD_HcGetAllMatches_selectMLS ( ZSTD_CCtx* zc, const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml) @@ -396,7 +396,7 @@ FORCE_INLINE U32 ZSTD_HcGetAllMatches_selectMLS ( } } -FORCE_INLINE U32 ZSTD_HcGetAllMatches_selectMLS_extDict ( +static U32 ZSTD_HcGetAllMatches_selectMLS_extDict ( ZSTD_CCtx* zc, const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml) @@ -769,7 +769,6 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set } - FORCE_INLINE void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const void* src, size_t srcSize, @@ -803,7 +802,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const U32 sufficient_len = ctx->params.targetLength; const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt); - /* init */ ZSTD_resetSeqStore(seqStorePtr); if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; @@ -1055,14 +1053,12 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set cur -= mlen; } - for (u = 0; u <= last_pos;) { + for (u = 0; u <= last_pos; ) { ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2); u += opt[u].mlen; } - cur = 0; - - while (cur < last_pos) { + for (cur=0; cur < last_pos; ) { U32 litLength; ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); mlen = opt[cur].mlen; diff --git a/programs/paramgrill.c b/programs/paramgrill.c index b77f7a38c..5ba3afab2 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -930,7 +930,7 @@ int optimizeForSize(char* inFileName) } -int usage(char* exename) +static int usage(char* exename) { DISPLAY( "Usage :\n"); DISPLAY( " %s [arg] file\n", exename); @@ -940,16 +940,17 @@ int usage(char* exename) return 0; } -int usage_advanced(void) +static int usage_advanced(void) { DISPLAY( "\nAdvanced options :\n"); DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); DISPLAY( " -B# : cut input into blocks of size # (default : single block)\n"); DISPLAY( " -P# : generated sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100); + DISPLAY( " -S : Single run\n"); return 0; } -int badusage(char* exename) +static int badusage(char* exename) { DISPLAY("Wrong parameters\n"); usage(exename); @@ -1064,13 +1065,16 @@ int main(int argc, char** argv) while ((*argument>= '0') && (*argument<='9')) g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0'; continue; - case 't': /* strategy */ - g_params.strategy = (ZSTD_strategy)0; + case 't': /* target length */ + g_params.targetLength = 0; argument++; - while ((*argument>= '0') && (*argument<='9')) { - g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy *10); - g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy + *argument++ - '0'); - } + while ((*argument>= '0') && (*argument<='9')) + g_params.targetLength *= 10, g_params.targetLength += *argument++ - '0'; + continue; + case 'S': /* strategy */ + argument++; + while ((*argument>= '0') && (*argument<='9')) + g_params.strategy = (ZSTD_strategy)(*argument++ - '0'); continue; case 'L': {