lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-11-28 23:23:59 +03:00
Code Activity

first implementation of small window size for btopt

Browse Source 
noticeably improves compression ratio
when window size is small (< 18).

enwik7	level 19

windowLog	`dev`	`smallwlog`	improvement
23	3.577	3.577	0.02%
22	3.536	3.538	0.06%
21	3.462	3.467	0.14%
20	3.364	3.377	0.39%
19	3.244	3.272	0.86%
18	3.110	3.166	1.80%
17	2.843	3.057	7.53%
16	2.724	2.943	8.04%
15	2.594	2.822	8.79%
14	2.456	2.686	9.36%
13	2.312	2.523	9.13%
12	2.162	2.361	9.20%
11	2.003	2.182	8.94%
This commit is contained in:
Yann Collet
2019-05-31 15:55:12 -07:00
parent b13a9207f9
commit bc601bdc6d
3 changed files with 42 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
lib/compress/zstd_compress_internal.h
View File

@@ -141,7 +141,7 @@ struct ZSTD_matchState_t {
U32* hashTable3;
U32* chainTable;
optState_t opt; /* optimal parser state */
const ZSTD_matchState_t * dictMatchState;
const ZSTD_matchState_t* dictMatchState;
ZSTD_compressionParameters cParams;
};
@@ -731,6 +731,38 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
}
}
MEM_STATIC void
ZSTD_checkDictValidity(ZSTD_window_t* window,
const void* blockEnd,
U32 maxDist,
U32* loadedDictEndPtr,
const ZSTD_matchState_t** dictMatchStatePtr)
{
U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
(unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
/* - When there is no dictionary : loadedDictEnd == 0.
In which case, the test (blockEndIdx > maxDist) is merely to avoid
overflowing next operation `newLowLimit = blockEndIdx - maxDist`.
- When there is a standard dictionary :
Index referential is copied from the dictionary,
which means it starts from 0.
In which case, loadedDictEnd == dictSize,
and it makes sense to compare `blockEndIdx > maxDist + dictSize`
since `blockEndIdx` also starts from zero.
- When there is an attached dictionary :
loadedDictEnd is expressed within the referential of the context,
so it can be directly compared against blockEndIdx.
*/
if (loadedDictEnd && (blockEndIdx > maxDist + loadedDictEnd)) {
/* On reaching window size, dictionaries are invalidated */
if (loadedDictEndPtr) *loadedDictEndPtr = 0;
if (dictMatchStatePtr) *dictMatchStatePtr = NULL;
}
}
/**
* ZSTD_window_update():
* Updates the window by appending [src, src + srcSize) to the window.