lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-11-30 11:21:33 +03:00
Code Activity

opt: minor improvements to log traces

Browse Source 
slight improvement when using fractional-bit evaluation (opt:dictionay)
This commit is contained in:
Yann Collet
2018-05-09 15:46:11 -07:00
parent c39061cb7b
commit ac6105463a
3 changed files with 34 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lib/compress/fse_compress.c
View File

@@ -161,10 +161,10 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
} } } } } } } }
#if 0 /* debug : symbol costs */ #if 0 /* debug : symbol costs */
DEBUGLOG(2, "\n --- table statistics : "); DEBUGLOG(5, "\n --- table statistics : ");
{ U32 symbol; { U32 symbol;
for (symbol=0; symbol<=maxSymbolValue; symbol++) { for (symbol=0; symbol<=maxSymbolValue; symbol++) {
DEBUGLOG(2, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f",
symbol, normalizedCounter[symbol], symbol, normalizedCounter[symbol],
FSE_getMaxNbBits(symbolTT, symbol), FSE_getMaxNbBits(symbolTT, symbol),
(double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256);

10
lib/compress/zstd_compress.c
View File

@@ -3450,7 +3450,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 22, 21, 22, 4, 5, 48, ZSTD_btopt }, /* level 16 */ { 22, 21, 22, 4, 5, 48, ZSTD_btopt }, /* level 16 */
{ 23, 22, 22, 4, 4, 48, ZSTD_btopt }, /* level 17 */ { 23, 22, 22, 4, 4, 48, ZSTD_btopt }, /* level 17 */
{ 23, 22, 22, 5, 3, 64, ZSTD_btopt }, /* level 18 */ { 23, 22, 22, 5, 3, 64, ZSTD_btopt }, /* level 18 */
{ 23, 23, 22, 7, 3,128, ZSTD_btultra }, /* level 19 */ { 23, 23, 22, 7, 3,128, ZSTD_btopt }, /* level 19 */
{ 25, 25, 23, 7, 3,128, ZSTD_btultra }, /* level 20 */ { 25, 25, 23, 7, 3,128, ZSTD_btultra }, /* level 20 */
{ 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */ { 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */
{ 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */ { 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */
@@ -3476,7 +3476,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/ { 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/
{ 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/
{ 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/
{ 18, 19, 18, 9, 3,256, ZSTD_btultra }, /* level 19.*/ { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/
{ 18, 19, 18, 11, 3,512, ZSTD_btultra }, /* level 20.*/ { 18, 19, 18, 11, 3,512, ZSTD_btultra }, /* level 20.*/
{ 18, 19, 18, 12, 3,512, ZSTD_btultra }, /* level 21.*/ { 18, 19, 18, 12, 3,512, ZSTD_btultra }, /* level 21.*/
{ 18, 19, 18, 13, 3,512, ZSTD_btultra }, /* level 22.*/ { 18, 19, 18, 13, 3,512, ZSTD_btultra }, /* level 22.*/
@@ -3502,7 +3502,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 17, 18, 17, 7, 3, 32, ZSTD_btopt }, /* level 16.*/ { 17, 18, 17, 7, 3, 32, ZSTD_btopt }, /* level 16.*/
{ 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ { 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/
{ 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/ { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/
{ 17, 18, 17, 7, 3,256, ZSTD_btultra }, /* level 19.*/ { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/
{ 17, 18, 17, 9, 3,256, ZSTD_btultra }, /* level 20.*/ { 17, 18, 17, 9, 3,256, ZSTD_btultra }, /* level 20.*/
{ 17, 18, 17, 10, 3,256, ZSTD_btultra }, /* level 21.*/ { 17, 18, 17, 10, 3,256, ZSTD_btultra }, /* level 21.*/
{ 17, 18, 17, 11, 3,512, ZSTD_btultra }, /* level 22.*/ { 17, 18, 17, 11, 3,512, ZSTD_btultra }, /* level 22.*/
@@ -3528,10 +3528,10 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/ { 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/
{ 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ { 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/
{ 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/ { 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/
{ 14, 15, 15, 6, 3,256, ZSTD_btultra }, /* level 19.*/ { 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/
{ 14, 15, 15, 8, 3,256, ZSTD_btultra }, /* level 20.*/ { 14, 15, 15, 8, 3,256, ZSTD_btultra }, /* level 20.*/
{ 14, 15, 15, 9, 3,256, ZSTD_btultra }, /* level 21.*/ { 14, 15, 15, 9, 3,256, ZSTD_btultra }, /* level 21.*/
{ 14, 15, 15, 10, 3,256, ZSTD_btultra }, /* level 22.*/ { 14, 15, 15, 10, 3,512, ZSTD_btultra }, /* level 22.*/
}, },
}; };

39
lib/compress/zstd_opt.c
View File

@@ -101,6 +101,12 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
# define BITCOST_SYMBOL(t,l,s) FSE_bitCost(t,l,s,BITCOST_ACCURACY) # define BITCOST_SYMBOL(t,l,s) FSE_bitCost(t,l,s,BITCOST_ACCURACY)
#endif #endif
MEM_STATIC double
ZSTD_fCost(U32 price)
{
return (double)price / (BITCOST_MULTIPLIER*8);
}
/* ZSTD_rawLiteralsCost() : /* ZSTD_rawLiteralsCost() :
* cost of literals (only) in specified segment (which length can be 0). * cost of literals (only) in specified segment (which length can be 0).
* does not include cost of literalLength symbol */ * does not include cost of literalLength symbol */
@@ -433,7 +439,7 @@ void ZSTD_updateTree_internal(
const BYTE* const base = ms->window.base; const BYTE* const base = ms->window.base;
U32 const target = (U32)(ip - base); U32 const target = (U32)(ip - base);
U32 idx = ms->nextToUpdate; U32 idx = ms->nextToUpdate;
DEBUGLOG(7, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)", DEBUGLOG(8, "ZSTD_updateTree_internal, from %u to %u (extDict:%u)",
idx, target, extDict); idx, target, extDict);
while(idx < target) while(idx < target)
@@ -481,7 +487,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
U32 nbCompares = 1U << cParams->searchLog; U32 nbCompares = 1U << cParams->searchLog;
size_t bestLength = lengthToBeat-1; size_t bestLength = lengthToBeat-1;
DEBUGLOG(7, "ZSTD_insertBtAndGetAllMatches"); DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches");
/* check repCode */ /* check repCode */
{ U32 const lastR = ZSTD_REP_NUM + ll0; { U32 const lastR = ZSTD_REP_NUM + ll0;
@@ -612,7 +618,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
ZSTD_match_t* matches, U32 const lengthToBeat) ZSTD_match_t* matches, U32 const lengthToBeat)
{ {
U32 const matchLengthSearch = cParams->searchLength; U32 const matchLengthSearch = cParams->searchLength;
DEBUGLOG(7, "ZSTD_BtGetAllMatches"); DEBUGLOG(8, "ZSTD_BtGetAllMatches");
if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */
ZSTD_updateTree_internal(ms, cParams, ip, iHighLimit, matchLengthSearch, extDict); ZSTD_updateTree_internal(ms, cParams, ip, iHighLimit, matchLengthSearch, extDict);
switch(matchLengthSearch) switch(matchLengthSearch)
@@ -786,8 +792,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
for ( ; pos <= end ; pos++ ) { for ( ; pos <= end ; pos++ ) {
U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel); U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel);
U32 const sequencePrice = literalsPrice + matchPrice; U32 const sequencePrice = literalsPrice + matchPrice;
DEBUGLOG(7, "rPos:%u => set initial price : %u", DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
pos, sequencePrice); pos, ZSTD_fCost(sequencePrice));
opt[pos].mlen = pos; opt[pos].mlen = pos;
opt[pos].off = offset; opt[pos].off = offset;
opt[pos].litlen = litlen; opt[pos].litlen = litlen;
@@ -814,14 +820,18 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
} }
assert(price < 1000000000); /* overflow check */ assert(price < 1000000000); /* overflow check */
if (price <= opt[cur].price) { if (price <= opt[cur].price) {
DEBUGLOG(7, "rPos:%u : better price (%u<%u) using literal", DEBUGLOG(7, "rPos:%u : better price (%.2f<=%.2f) using literal",
cur, price, opt[cur].price); cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price));
opt[cur].mlen = 1; opt[cur].mlen = 1;
opt[cur].off = 0; opt[cur].off = 0;
opt[cur].litlen = litlen; opt[cur].litlen = litlen;
opt[cur].price = price; opt[cur].price = price;
memcpy(opt[cur].rep, opt[cur-1].rep, sizeof(opt[cur].rep)); memcpy(opt[cur].rep, opt[cur-1].rep, sizeof(opt[cur].rep));
} } } else {
DEBUGLOG(7, "rPos:%u : literal would cost more (%.2f>%.2f)",
cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price));
}
}
/* last match must start at a minimum distance of 8 from oend */ /* last match must start at a minimum distance of 8 from oend */
if (inr > ilimit) continue; if (inr > ilimit) continue;
@@ -829,8 +839,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
if (cur == last_pos) break; if (cur == last_pos) break;
if ( (optLevel==0) /*static_test*/ if ( (optLevel==0) /*static_test*/
&& (opt[cur+1].price <= opt[cur].price) ) && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) {
DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1);
continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */
}
{ U32 const ll0 = (opt[cur].mlen != 1); { U32 const ll0 = (opt[cur].mlen != 1);
U32 const litlen = (opt[cur].mlen == 1) ? opt[cur].litlen : 0; U32 const litlen = (opt[cur].mlen == 1) ? opt[cur].litlen : 0;
@@ -838,7 +850,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
U32 const basePrice = previousPrice + ZSTD_fullLiteralsCost(inr-litlen, litlen, optStatePtr); U32 const basePrice = previousPrice + ZSTD_fullLiteralsCost(inr-litlen, litlen, optStatePtr);
U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, inr, iend, extDict, opt[cur].rep, ll0, matches, minMatch); U32 const nbMatches = ZSTD_BtGetAllMatches(ms, cParams, inr, iend, extDict, opt[cur].rep, ll0, matches, minMatch);
U32 matchNb; U32 matchNb;
if (!nbMatches) continue; if (!nbMatches) {
DEBUGLOG(7, "rPos:%u : no match found", cur);
continue;
}
{ U32 const maxML = matches[nbMatches-1].len; { U32 const maxML = matches[nbMatches-1].len;
DEBUGLOG(7, "rPos:%u, found %u matches, of maxLength=%u", DEBUGLOG(7, "rPos:%u, found %u matches, of maxLength=%u",
@@ -868,8 +883,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); int const price = basePrice + ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel);
if ((pos > last_pos) || (price < opt[pos].price)) { if ((pos > last_pos) || (price < opt[pos].price)) {
DEBUGLOG(7, "rPos:%u => new better price (%u<%u)", DEBUGLOG(7, "rPos:%u => new better price (%.2f<%.2f)",
pos, price, opt[pos].price); pos, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price));
while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */ while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */
opt[pos].mlen = mlen; opt[pos].mlen = mlen;
opt[pos].off = offset; opt[pos].off = offset;