lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-11-30 11:21:33 +03:00
Code Activity

opt: removed static prices

Browse Source 
after testing, it's actually always better to use dynamic prices
albeit initialised from dictionary.
This commit is contained in:
Yann Collet
2018-05-14 18:04:08 -07:00
parent 761758982e
commit 2c26df0e13
2 changed files with 3 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/compress/zstd_compress_internal.h
View File

@@ -76,7 +76,7 @@ typedef struct {
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_NUM];
} ZSTD_optimal_t; } ZSTD_optimal_t;
typedef enum { zop_dynamic=0, zop_predef, zop_static } ZSTD_OptPrice_e; typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e;
typedef struct { typedef struct {
/* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */

42
lib/compress/zstd_opt.c
View File

@@ -40,14 +40,9 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
assert(optPtr->symbolCosts != NULL); assert(optPtr->symbolCosts != NULL);
if (optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */ if (optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
if (srcSize <= 8192) /* heuristic */ optPtr->priceType = zop_dynamic;
optPtr->priceType = zop_static;
else {
assert(optPtr->priceType == zop_dynamic);
}
assert(optPtr->litFreq != NULL); assert(optPtr->litFreq != NULL);
assert(optPtr->symbolCosts != NULL);
optPtr->litSum = 0; optPtr->litSum = 0;
{ unsigned lit; { unsigned lit;
for (lit=0; lit<=MaxLit; lit++) { for (lit=0; lit<=MaxLit; lit++) {
@@ -156,7 +151,7 @@ static void ZSTD_rescaleFreqs(optState_t* const optPtr,
#if 1 /* approximation at bit level */ #if 1 /* approximation at bit level */
# define BITCOST_ACCURACY 0 # define BITCOST_ACCURACY 0
# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
# define BITCOST_SYMBOL(t,l,s) ((void)l, FSE_getMaxNbBits(t,s)*BITCOST_MULTIPLIER) # define BITCOST_SYMBOL(t,l,s) ((void)(l), FSE_getMaxNbBits(t,s) * BITCOST_MULTIPLIER)
#else /* fractional bit accuracy */ #else /* fractional bit accuracy */
# define BITCOST_ACCURACY 8 # define BITCOST_ACCURACY 8
# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
@@ -177,14 +172,6 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
{ {
if (litLength == 0) return 0; if (litLength == 0) return 0;
if (optPtr->priceType == zop_predef) return (litLength*6); /* 6 bit per literal - no statistic used */ if (optPtr->priceType == zop_predef) return (litLength*6); /* 6 bit per literal - no statistic used */
if (optPtr->priceType == zop_static) {
U32 u, cost;
assert(optPtr->symbolCosts != NULL);
assert(optPtr->symbolCosts->hufCTable_repeatMode == HUF_repeat_valid);
for (u=0, cost=0; u < litLength; u++)
cost += HUF_getNbBits(optPtr->symbolCosts->hufCTable, literals[u]);
return cost * BITCOST_MULTIPLIER;
}
/* dynamic statistics */ /* dynamic statistics */
{ U32 u; { U32 u;
@@ -199,14 +186,6 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
* cost of literalLength symbol */ * cost of literalLength symbol */
static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr) static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr)
{ {
if (optPtr->priceType == zop_static) {
U32 const llCode = ZSTD_LLcode(litLength);
FSE_CState_t cstate;
FSE_initCState(&cstate, optPtr->symbolCosts->litlengthCTable);
{ U32 const price = LL_bits[llCode]*BITCOST_MULTIPLIER + BITCOST_SYMBOL(cstate.symbolTT, cstate.stateLog, llCode);
DEBUGLOG(8, "ZSTD_litLengthPrice: ll=%u, bitCost=%.2f", litLength, (double)price / BITCOST_MULTIPLIER);
return price;
} }
if (optPtr->priceType == zop_predef) return ZSTD_highbit32((U32)litLength+1); if (optPtr->priceType == zop_predef) return ZSTD_highbit32((U32)litLength+1);
/* dynamic statistics */ /* dynamic statistics */
@@ -231,14 +210,6 @@ static U32 ZSTD_fullLiteralsCost(const BYTE* const literals, U32 const litLength
* to provide a cost which is directly comparable to a match ending at same position */ * to provide a cost which is directly comparable to a match ending at same position */
static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr) static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr)
{ {
if (optPtr->priceType == zop_static) {
U32 const llCode = ZSTD_LLcode(litLength);
FSE_CState_t cstate;
FSE_initCState(&cstate, optPtr->symbolCosts->litlengthCTable);
return (int)(LL_bits[llCode] * BITCOST_MULTIPLIER)
+ BITCOST_SYMBOL(cstate.symbolTT, cstate.stateLog, llCode)
- BITCOST_SYMBOL(cstate.symbolTT, cstate.stateLog, 0);
}
if (optPtr->priceType >= zop_predef) return ZSTD_highbit32(litLength+1); if (optPtr->priceType >= zop_predef) return ZSTD_highbit32(litLength+1);
/* dynamic statistics */ /* dynamic statistics */
@@ -281,15 +252,6 @@ ZSTD_getMatchPrice(U32 const offset, U32 const matchLength,
U32 const mlBase = matchLength - MINMATCH; U32 const mlBase = matchLength - MINMATCH;
assert(matchLength >= MINMATCH); assert(matchLength >= MINMATCH);
if (optPtr->priceType == zop_static) {
U32 const mlCode = ZSTD_MLcode(mlBase);
FSE_CState_t mlstate, offstate;
FSE_initCState(&mlstate, optPtr->symbolCosts->matchlengthCTable);
FSE_initCState(&offstate, optPtr->symbolCosts->offcodeCTable);
return BITCOST_SYMBOL(offstate.symbolTT, offstate.stateLog, offCode) + offCode*BITCOST_MULTIPLIER
+ BITCOST_SYMBOL(mlstate.symbolTT, mlstate.stateLog, mlCode) + ML_bits[mlCode]*BITCOST_MULTIPLIER;
}
if (optPtr->priceType == zop_predef) /* fixed scheme, do not use statistics */ if (optPtr->priceType == zop_predef) /* fixed scheme, do not use statistics */
return ZSTD_highbit32(mlBase+1) + 16 + offCode; return ZSTD_highbit32(mlBase+1) + 16 + offCode;