lib/lz4
1
0
Fork 0
mirror of https://github.com/lz4/lz4.git synced 2026-10-24 01:48:57 +03:00
Code Activity

lz4: clear the table for a new stream starting with a large block

Browse Source 
LZ4_resetStream_fast() keeps the hash table, since it doesn't know the stream's first block size.
Reusing a table means rejecting its stale entries,
a poorly predictable branch, which costs more than clearing the table beyond a small input size.
It also made the output depend on the state's history :
a cleared table tests position 0 for unused hash slots, a used table rejects its stale entries.

LZ4_compress_fast_continue() now clears the table when a stream without history
(no dictionary) starts with a block >= LZ4_CLEAR_TABLE_THRESHOLD,
the threshold already used by one-shot fast reset (LZ4_HASHTABLESIZE / 16, 1 KB by default).
Such streams are faster, and produce the same output as a new stream.
Smaller first blocks, and streams using a dictionary, are unchanged.
LZ4F is unchanged : its linked frames already start from a cleared table.

LZ4_resetStream_fast() + LZ4_compress_fast_continue() on a reused state,
MB/s, AMD EPYC Genoa, gcc 11 -O3 (aligned code, both builds) :

  first block    silesia.tar           enwik9
  default table (LZ4_MEMORY_USAGE=14), threshold 1 KB :
     < 1 KB      unchanged             unchanged
       1 KB      572 -> 665  (+16%)    388 -> 520  (+34%)
       2 KB      479 -> 667  (+39%)    315 -> 500  (+59%)
       4 KB      403 -> 655  (+63%)    257 -> 478  (+86%)
       8 KB      371 -> 643  (+73%)    238 -> 462  (+94%)
      16 KB      396 -> 633  (+60%)    264 -> 443  (+68%)
      64 KB      501 -> 620  (+24%)    341 -> 419  (+23%)
  other tables, beyond their threshold :
    4 KB table (256 B)      +12% to +102%         +7% to +133%
   64 KB table (4 KB)       +10% to +46%          +20% to +56%
  256 KB table (16 KB)       +2% to +25%           +7% to +26%
This commit is contained in:
Yann Collet
2026-10-09 12:47:19 -07:00
parent 3d8a138ab7
commit e13b541638
1 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
lib/lz4.c
View File

@@ -889,6 +889,10 @@ LZ4_getPosition(const BYTE* p,
return LZ4_getPositionOnHash(h, tableBase, tableType);
}
/* beyond this input size (1 KB by default),
* clearing the table is faster than rejecting its stale entries */
#define LZ4_CLEAR_TABLE_THRESHOLD (LZ4_HASHTABLESIZE / 16)
LZ4_FORCE_INLINE void
LZ4_prepareTable(LZ4_stream_t_internal* const cctx,
const int inputSize,
@@ -899,13 +903,11 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx,
*/
if ((tableType_t)cctx->tableType != clearedTable) {
assert(inputSize >= 0);
/* beyond LZ4_HASHTABLESIZE / 16 (1 KB by default),
* clearing the table is faster than rejecting its stale entries */
if ((tableType_t)cctx->tableType != tableType
|| ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU)
|| ((tableType == byU32) && cctx->currentOffset > 1 GB)
|| tableType == byPtr
|| inputSize >= LZ4_HASHTABLESIZE / 16)
|| inputSize >= LZ4_CLEAR_TABLE_THRESHOLD)
{
DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", (void*)cctx);
MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE);
@@ -1726,6 +1728,10 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream,
DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize);
/* new stream (no history) : clear the table beyond LZ4_CLEAR_TABLE_THRESHOLD */
if ((streamPtr->dictSize == 0) && (streamPtr->dictCtx == NULL) && (inputSize >= LZ4_CLEAR_TABLE_THRESHOLD))
LZ4_prepareTable(streamPtr, inputSize, byU32);
LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */
if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT;
if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX;