lib/lz4
1
0
Fork 0
mirror of https://github.com/lz4/lz4.git synced 2026-10-24 01:48:57 +03:00
Code Activity

tests: move the simple block unit tests from fuzzer.c to unittest

Browse Source 
33 tests, in 2 new suites : lz4 (decoding of invalid input, one-shot and streaming compression)
and lz4hc (streaming, dictionaries, destSize). Tests depending on the HC level run on both levels
previously selected by fuzzer.

UT_sample(), KB, MB and ARRAY_SIZE() are now provided by unittest.h, for all suites.
This commit is contained in:
Yann Collet
2026-10-22 17:34:15 -07:00
parent 9af9ec8f26
commit 5d63c34ac0
8 changed files with 837 additions and 728 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build/meson/meson/tests/meson.build
View File

@@ -71,6 +71,8 @@ test_exes = {
'unittest': {
'sources': files(
lz4_source_root / 'tests/unittest.c',
lz4_source_root / 'tests/unittest_lz4.c',
lz4_source_root / 'tests/unittest_lz4hc.c',
lz4_source_root / 'tests/unittest_lz4frame.c',
lz4_source_root / 'tests/datagen.c',
),

2
tests/Makefile
View File

@@ -125,7 +125,7 @@ frametest:
$(eval $(call c_program_shared_o,frametest,lz4frame.o lz4file.o lz4.o lz4hc.o xxhash.o frametest.o))
unittest:
$(eval $(call c_program_shared_o,unittest,lz4frame.o lz4.o lz4hc.o xxhash.o datagen.o unittest.o unittest_lz4frame.o))
$(eval $(call c_program_shared_o,unittest,lz4frame.o lz4.o lz4hc.o xxhash.o datagen.o unittest.o unittest_lz4.o unittest_lz4hc.o unittest_lz4frame.o))
roundTripTest:
$(eval $(call c_program_shared_o,roundTripTest,lz4.o lz4hc.o xxhash.o roundTripTest.o))

702
tests/fuzzer.c
View File

@@ -1202,159 +1202,6 @@ static void FUZ_unitTests(int compressionLevel)
/* 32-bits address space overflow test */
FUZ_AddressOverflow();
/* Test decoding with empty input */
DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input \n");
LZ4_decompress_safe(testCompressed, testVerify, 0, testInputSize);
/* Test decoding with a one byte input */
DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input \n");
{ char const tmp = (char)0xFF;
LZ4_decompress_safe(&tmp, testVerify, 1, testInputSize);
}
/* Test decoding shortcut edge case */
DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case \n");
{ char tmp[17];
/* 14 bytes of literals, followed by a 14 byte match.
* Should not read beyond the end of the buffer.
* See https://github.com/lz4/lz4/issues/508. */
*tmp = (char)0xEE;
memset(tmp + 1, 0, 14);
tmp[15] = 14;
tmp[16] = 0;
{ int const r = LZ4_decompress_safe(tmp, testVerify, sizeof(tmp), testInputSize);
FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail");
} }
/* Test decoding with all 0xFF input - OOB read regression test */
DISPLAYLEVEL(3, "LZ4_decompress_safe() with all-0xFF input \n");
{ char tmp[31];
memset(tmp, 0xFF, sizeof(tmp));
{ int const r = LZ4_decompress_safe(tmp, testVerify, sizeof(tmp), testInputSize);
FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail on all-0xFF input");
} }
/* Test that ip does not go beyond iend in the fast decode loop.
* The block has 14 literals, ML_MASK match token, offset=14, one extension byte=0.
* This positions ip exactly at iend when read_variable_length reads the extension,
* then goto safe_match_copy would loop back with ip==iend, triggering assert(ip<iend).
* Output buffer is 65 bytes: large enough to enter the fast loop (>=64),
* small enough that op+length >= oend-64 triggers the safe_match_copy path. */
DISPLAYLEVEL(3, "LZ4_decompress_safe() fast-loop ip-at-iend regression test \n");
{ char block[18];
char out[65];
block[0] = (char)0xEF; /* 14 literals, ML_MASK match */
memset(block + 1, 'A', 14); /* 14 literal bytes */
block[15] = 0x0E; block[16] = 0x00; /* offset = 14 */
block[17] = 0x00; /* match length extension = 0 */
{ int const r = LZ4_decompress_safe(block, out, sizeof(block), sizeof(out));
FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail on this crafted input");
} }
/* Test that ip does not go beyond iend in the safe decode loop shortcut path.
* Same block as above, but output buffer is 40 bytes: small enough to skip
* the fast loop (<64), large enough for the shortcut (>=32).
* The shortcut reads literals+offset, landing ip at iend-1, then goto _copy_match
* calls read_variable_length with ip >= ilimit. */
DISPLAYLEVEL(3, "LZ4_decompress_safe() safe-loop shortcut ip-at-iend regression test \n");
{ char block[18];
char out[40];
block[0] = (char)0xEF; /* 14 literals, ML_MASK match */
memset(block + 1, 'A', 14); /* 14 literal bytes */
block[15] = 0x0E; block[16] = 0x00; /* offset = 14 */
block[17] = 0x00; /* match length extension = 0 */
{ int const r = LZ4_decompress_safe(block, out, sizeof(block), sizeof(out));
FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail on this crafted input");
} }
/* Invalid offset 0 must not leave previous output buffer content in decoded data
* (see doc/lz4_Block_format.md). 1 literal, then a match at offset 0,
* partially decoded into a 20-byte target which ends within the match. */
DISPLAYLEVEL(3, "LZ4_decompress_safe_partial() with offset 0 \n");
{ char block[32];
char out[20];
memset(block, 0, sizeof(block));
block[0] = 0x1F; /* 1 literal, ML_MASK match */
block[1] = 'A'; /* literal */
block[2] = 0x00; block[3] = 0x00; /* offset = 0 (invalid) */
block[4] = 64; /* match length extension */
memset(out, 0xAA, sizeof(out));
{ int const r = LZ4_decompress_safe_partial(block, out, sizeof(block), sizeof(out), sizeof(out));
int i;
for (i = 1; i < r; i++) {
FUZ_CHECKTEST(out[i] == (char)0xAA, "LZ4_decompress_safe_partial() kept previous output content at position %i, with offset 0", i);
} } }
/* to be tested with undefined sanitizer */
DISPLAYLEVEL(3, "LZ4_compress_default() with NULL input:");
{ int const maxCSize = LZ4_compressBound(0);
int const cSize = LZ4_compress_default(NULL, testCompressed, 0, maxCSize);
FUZ_CHECKTEST(!(cSize==1 && testCompressed[0]==0),
"compressing empty should give byte 0"
" (maxCSize == %i) (cSize == %i) (byte == 0x%02X)",
maxCSize, cSize, testCompressed[0]);
}
DISPLAYLEVEL(3, " OK \n");
DISPLAYLEVEL(3, "LZ4_compress_default() with both NULL input and output:");
{ int const cSize = LZ4_compress_default(NULL, NULL, 0, 0);
FUZ_CHECKTEST(cSize != 0,
"compressing into NULL must fail"
" (cSize == %i != 0)", cSize);
}
DISPLAYLEVEL(3, " OK \n");
DISPLAYLEVEL(3, "LZ4_compress_destSize_extState() with out-of-range acceleration : ");
{ static const int accels[] = { 0, -1, -100, INT_MIN, INT_MAX };
int const srcSize = 128 KB;
int const targetDstSize = srcSize / 4; /* < LZ4_compressBound(srcSize) : fillOutput mode */
LZ4_stream_t state;
int refConsumed = srcSize, refSize;
U32 refCrc;
size_t n;
refSize = LZ4_compress_destSize_extState(&state, testInput, testCompressed, &refConsumed, targetDstSize, 1);
FUZ_CHECKTEST(refSize <= 0, "LZ4_compress_destSize_extState() failed");
refCrc = XXH32(testCompressed, (size_t)refSize, 0);
for (n = 0; n < sizeof(accels)/sizeof(accels[0]); n++) {
int consumed = srcSize;
int const cSize = LZ4_compress_destSize_extState(&state, testInput, testCompressed, &consumed, targetDstSize, accels[n]);
FUZ_CHECKTEST(cSize <= 0 || cSize > targetDstSize, "acceleration %i : compression failed (%i)", accels[n], cSize);
if (accels[n] < 1) { /* replaced by the default acceleration, 1 */
FUZ_CHECKTEST(cSize != refSize || consumed != refConsumed || XXH32(testCompressed, (size_t)cSize, 0) != refCrc,
"acceleration %i : output differs from acceleration 1", accels[n]);
}
{ int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize);
FUZ_CHECKTEST(dSize != consumed || memcmp(testInput, testVerify, (size_t)consumed),
"acceleration %i : round-trip failure", accels[n]);
} }
}
DISPLAYLEVEL(3, "OK \n");
/* in-place compression test */
DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :");
{ int const sampleSize = 65 KB;
int const maxCSize = LZ4_COMPRESSBOUND(sampleSize);
int const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize);
int const startInputIndex = outSize - sampleSize;
char* const startInput = testCompressed + startInputIndex;
XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0);
int cSize;
assert(outSize < (int)testCompressedSize);
memcpy(startInput, testInput, sampleSize); /* copy at end of buffer */
/* compress in-place */
cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize);
assert(cSize != 0); /* ensure compression is successful */
assert(maxCSize < INT_MAX);
assert(cSize <= maxCSize);
/* decompress and verify */
{ int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize);
assert(dSize == sampleSize); /* correct size */
{ XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0);
FUZ_CHECKTEST(crcCheck != crcOrig, "LZ4_decompress_safe decompression corruption");
} } }
DISPLAYLEVEL(3, " OK \n");
/* in-place decompression test */
DISPLAYLEVEL(3, "in-place decompression, limit case:");
{ int const sampleSize = 65 KB;
@@ -1379,220 +1226,9 @@ static void FUZ_unitTests(int compressionLevel)
} } } } }
DISPLAYLEVEL(3, " OK \n");
DISPLAYLEVEL(3, "LZ4_initStream with multiple valid alignments : ");
{ typedef struct {
LZ4_stream_t state1;
LZ4_stream_t state2;
char c;
LZ4_stream_t state3;
} shct;
shct* const shc = (shct*)malloc(sizeof(*shc));
assert(shc != NULL);
memset(shc, 0, sizeof(*shc));
DISPLAYLEVEL(4, "state1(%p) state2(%p) state3(%p) LZ4_stream_t size(0x%x): ",
(void*)&(shc->state1), (void*)&(shc->state2), (void*)&(shc->state3), (unsigned)sizeof(LZ4_stream_t));
FUZ_CHECKTEST( LZ4_initStream(&(shc->state1), sizeof(shc->state1)) == NULL, "state1 (%p) failed init", (void*)&(shc->state1) );
FUZ_CHECKTEST( LZ4_initStream(&(shc->state2), sizeof(shc->state2)) == NULL, "state2 (%p) failed init", (void*)&(shc->state2) );
FUZ_CHECKTEST( LZ4_initStream(&(shc->state3), sizeof(shc->state3)) == NULL, "state3 (%p) failed init", (void*)&(shc->state3) );
FUZ_CHECKTEST( LZ4_initStream((char*)&(shc->state1) + 1, sizeof(shc->state1)) != NULL,
"hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->state1) + 1) );
free(shc);
}
DISPLAYLEVEL(3, "all inits OK \n");
/* Allocation test */
{ LZ4_stream_t* const statePtr = LZ4_createStream();
FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
LZ4_freeStream(statePtr);
}
/* LZ4 streaming tests */
{ LZ4_stream_t streamingState;
/* simple compression test */
LZ4_initStream(&streamingState, sizeof(streamingState));
{ int const cs = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1);
FUZ_CHECKTEST(cs==0, "LZ4_compress_fast_continue() compression failed!");
{ int const r = LZ4_decompress_safe(testCompressed, testVerify, cs, testCompressedSize);
FUZ_CHECKTEST(r!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
} }
{ U64 const crcOrig = XXH64(testInput, testCompressedSize, 0);
U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
}
DISPLAYLEVEL(3, "LZ4_resetStream_fast() : a large first block gives the same output as a new stream : ");
{ int const blockSize = 64 KB; /* beyond the table clearing threshold, whatever LZ4_MEMORY_USAGE */
char* const block = (char*)malloc((size_t)blockSize);
int refSize, r;
U32 refCrc;
FUZ_CHECKTEST(block == NULL, "allocation failure");
memcpy(block, testInput + 64 KB, (size_t)blockSize);
/* the first 4 bytes reappear, with a different 5th byte :
* a match only found when starting from a cleared table */
memcpy(block, "ABCDx", 5);
memcpy(block + 16, "ABCDy", 5);
LZ4_initStream(&streamingState, sizeof(streamingState));
refSize = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1);
FUZ_CHECKTEST(refSize <= 0, "LZ4_compress_fast_continue() failed");
refCrc = XXH32(testCompressed, (size_t)refSize, 0);
LZ4_resetStream_fast(&streamingState); /* the state now has a history */
r = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1);
FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc,
"output differs from a new stream (%i vs %i bytes)", r, refSize);
LZ4_saveDict(&streamingState, testVerify, 0); /* drop history, without reset */
r = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1);
FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc,
"after LZ4_saveDict(,,0) : output differs from a new stream (%i vs %i bytes)", r, refSize);
free(block);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "periodic runs : chains of short matches must not hide the period (#1805) : ");
{ static const char dict[8] = { 0, 0, 0, 0, 0, 0, 0, 4 };
char page[8 + 4096]; /* dict, then a 4 KB page alternating runs of 16 x 0xFF and 16 x 0x00 */
LZ4_stream_t dictState;
int r;
memcpy(page, dict, sizeof(dict));
for (r = 0; r < 32; r++) { /* several rotations compressed to ~1.5 KB, instead of < 60 bytes */
int i, c;
for (i = 0; i < 4096; i++)
page[8 + i] = (((i + r) % 4096 + 8) / 16) % 2 == 0 ? (char)0xFF : 0;
c = LZ4_compress_default(page, testCompressed, (int)sizeof(page), (int)testCompressedSize);
FUZ_CHECKTEST(c <= 0 || c > 100, "rotation %i, dictionary bytes in the same block : %i bytes", r, c);
LZ4_initStream(&streamingState, sizeof(streamingState));
LZ4_loadDict(&streamingState, dict, (int)sizeof(dict));
c = LZ4_compress_fast_continue(&streamingState, page + 8, testCompressed, 4096, (int)testCompressedSize, 1);
FUZ_CHECKTEST(c <= 0 || c > 100, "rotation %i, LZ4_loadDict() : %i bytes", r, c);
LZ4_initStream(&dictState, sizeof(dictState));
LZ4_loadDict(&dictState, dict, (int)sizeof(dict));
LZ4_initStream(&streamingState, sizeof(streamingState));
LZ4_attach_dictionary(&streamingState, &dictState);
c = LZ4_compress_fast_continue(&streamingState, page + 8, testCompressed, 4096, (int)testCompressedSize, 1);
FUZ_CHECKTEST(c <= 0 || c > 100, "rotation %i, LZ4_attach_dictionary() : %i bytes", r, c);
}
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "LZ4_loadDict() with a dictionary too small to be used : a large first block gives the same output as a new stream : ");
{ int const blockSize = 64 KB; /* beyond the table clearing threshold, whatever LZ4_MEMORY_USAGE */
char* const block = (char*)malloc((size_t)blockSize);
LZ4_stream_t dictStream;
int refSize, dictSize;
U32 refCrc;
FUZ_CHECKTEST(block == NULL, "allocation failure");
memcpy(block, testInput + 64 KB, (size_t)blockSize);
/* a match only found when starting from a cleared table */
memcpy(block, "ABCDx", 5);
memcpy(block + 16, "ABCDy", 5);
LZ4_initStream(&streamingState, sizeof(streamingState));
refSize = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1);
FUZ_CHECKTEST(refSize <= 0, "LZ4_compress_fast_continue() failed");
refCrc = XXH32(testCompressed, (size_t)refSize, 0);
/* below HASH_UNIT on all platforms : LZ4_loadDict() ignores the dictionary */
for (dictSize = 0; dictSize < 4; dictSize++) {
int r;
LZ4_loadDict(&dictStream, testInput, dictSize);
r = LZ4_compress_fast_continue(&dictStream, block, testCompressed, blockSize, testCompressedSize, 1);
FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc,
"after LZ4_loadDict() (%i bytes) : output differs from a new stream (%i vs %i bytes)", dictSize, r, refSize);
LZ4_loadDict(&dictStream, testInput, dictSize);
LZ4_initStream(&streamingState, sizeof(streamingState));
LZ4_attach_dictionary(&streamingState, &dictStream);
r = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1);
FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc,
"after LZ4_attach_dictionary() (%i bytes) : output differs from a new stream (%i vs %i bytes)", dictSize, r, refSize);
}
free(block);
}
DISPLAYLEVEL(3, "OK \n");
/* early saveDict */
DISPLAYLEVEL(3, "saveDict (right after init) : ");
{ LZ4_stream_t* const ctx = LZ4_initStream(&streamingState, sizeof(streamingState));
assert(ctx != NULL); /* ensure init is successful */
/* Check access violation with asan */
FUZ_CHECKTEST( LZ4_saveDict(ctx, NULL, 0) != 0,
"LZ4_saveDict() can't save anything into (NULL,0)");
/* Check access violation with asan */
{ char tmp_buffer[240] = { 0 };
FUZ_CHECKTEST( LZ4_saveDict(ctx, tmp_buffer, sizeof(tmp_buffer)) != 0,
"LZ4_saveDict() can't save anything since compression hasn't started");
}
/* a negative size saves nothing (instead of wrapping to 64 KB),
* and the next block no longer references the previous dictionary :
* it copies the end of the dictionary, so it would match it */
{ char saveBuffer[16];
char* const block = (char*)malloc(4 KB);
int cSize, dSize;
FUZ_CHECKTEST(block == NULL, "allocation failed");
memcpy(block, testInput + 60 KB, 4 KB);
FUZ_CHECKTEST( LZ4_loadDict(ctx, testInput, 64 KB) <= 0,
"LZ4_loadDict() failed to prepare saveDict negative-size test");
FUZ_CHECKTEST( LZ4_saveDict(ctx, saveBuffer, -1) != 0,
"LZ4_saveDict() must save nothing with a negative size");
cSize = LZ4_compress_fast_continue(ctx, block, testCompressed, 4 KB, testCompressedSize, 1);
FUZ_CHECKTEST(cSize <= 0, "LZ4_compress_fast_continue() failed");
dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, 4 KB);
FUZ_CHECKTEST(dSize != 4 KB || memcmp(testVerify, block, 4 KB),
"after LZ4_saveDict() with a negative size, the next block references the previous dictionary");
free(block);
}
}
DISPLAYLEVEL(3, "OK \n");
/* the new block can overwrite the previous one (its dictionary) : any overlap must be handled */
DISPLAYLEVEL(3, "LZ4_compress_*_continue() with input overwriting the previous block : ");
{ /* block 1 : 1000 bytes at position 1000 ; block 2 : { position, size } */
static const int overlaps[4][2] = {
{ 500, 1000 }, /* covers the beginning of block 1 */
{ 1000, 1000 }, /* same place */
{ 500, 2000 }, /* covers block 1 entirely */
{ 1500, 1000 } }; /* covers the end of block 1 */
int const pos1 = 1000, size1 = 1000;
char* const src = testVerify + 64 KB; /* content of both blocks */
char* const cBlock2 = testCompressed + LZ4_COMPRESSBOUND(4 KB);
LZ4_streamHC_t* const sHC = LZ4_createStreamHC();
int ov, hc, trial;
FUZ_CHECKTEST(sHC == NULL, "LZ4_createStreamHC() allocation failed");
for (ov = 0; ov < 4; ov++)
for (hc = 0; hc < 2; hc++)
for (trial = 0; trial < 64; trial++) {
int const pos2 = overlaps[ov][0], size2 = overlaps[ov][1];
LZ4_streamDecode_t sd;
int c1, c2, i, r;
for (i = 0; i < size1 + size2; i++) src[i] = "ACGT"[FUZ_rand(&randState) & 3]; /* many matches */
LZ4_initStream(&streamingState, sizeof(streamingState));
LZ4_initStreamHC(sHC, sizeof(*sHC));
memcpy(ringBuffer + pos1, src, (size_t)size1);
c1 = hc ? LZ4_compress_HC_continue(sHC, ringBuffer + pos1, testCompressed, size1, LZ4_COMPRESSBOUND(4 KB))
: LZ4_compress_fast_continue(&streamingState, ringBuffer + pos1, testCompressed, size1, LZ4_COMPRESSBOUND(4 KB), 1);
memcpy(ringBuffer + pos2, src + size1, (size_t)size2);
c2 = hc ? LZ4_compress_HC_continue(sHC, ringBuffer + pos2, cBlock2, size2, LZ4_COMPRESSBOUND(4 KB))
: LZ4_compress_fast_continue(&streamingState, ringBuffer + pos2, cBlock2, size2, LZ4_COMPRESSBOUND(4 KB), 1);
FUZ_CHECKTEST(c1 <= 0 || c2 <= 0, "compression failed");
LZ4_setStreamDecode(&sd, NULL, 0);
r = LZ4_decompress_safe_continue(&sd, testCompressed, testVerify, c1, size1);
r += LZ4_decompress_safe_continue(&sd, cBlock2, testVerify + size1, c2, size2);
FUZ_CHECKTEST(r != size1 + size2 || memcmp(testVerify, src, (size_t)(size1 + size2)),
"%s : corrupted output, block 2 (position %i, size %i) overwrites block 1 (position %i, size %i)",
hc ? "LZ4_compress_HC_continue()" : "LZ4_compress_fast_continue()", pos2, size2, pos1, size1);
}
LZ4_freeStreamHC(sHC);
}
DISPLAYLEVEL(3, "OK \n");
/* ring buffer test */
{ XXH64_state_t xxhOrig;
@@ -1646,270 +1282,11 @@ static void FUZ_unitTests(int compressionLevel)
} }
}
DISPLAYLEVEL(3, "LZ4_compress_HC() with NULL input:");
{ int const maxCSize = LZ4_compressBound(0);
int level;
for (level=LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++) {
int const cSize = LZ4_compress_HC(NULL, testCompressed, 0, maxCSize, level);
FUZ_CHECKTEST(!(cSize==1 && testCompressed[0]==0),
"compressing empty should give byte 0"
" (maxCSize == %i) (cSize == %i) (byte == 0x%02X)",
maxCSize, cSize, testCompressed[0]);
}
}
DISPLAYLEVEL(3, " OK \n");
DISPLAYLEVEL(3, "LZ4_compress_HC() with both NULL input and output:");
{ int level;
for (level=LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++) {
int const cSize = LZ4_compress_HC(NULL, NULL, 0, 0, level);
FUZ_CHECKTEST(cSize != 0,
"compressing into NULL must fail"
" (cSize == %i != 0)", cSize);
}
}
DISPLAYLEVEL(3, " OK \n");
DISPLAYLEVEL(3, "LZ4_initStreamHC with multiple valid alignments : ");
{ typedef struct {
LZ4_streamHC_t hc1;
LZ4_streamHC_t hc2;
char c;
LZ4_streamHC_t hc3;
} shct;
shct* const shc = (shct*)malloc(sizeof(*shc));
assert(shc != NULL);
memset(shc, 0, sizeof(*shc));
DISPLAYLEVEL(4, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ",
(void*)&(shc->hc1), (void*)&(shc->hc2), (void*)&(shc->hc3),
(unsigned)sizeof(LZ4_streamHC_t));
FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc1), sizeof(shc->hc1)) == NULL, "hc1 (%p) failed init", (void*)&(shc->hc1) );
FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc2), sizeof(shc->hc2)) == NULL, "hc2 (%p) failed init", (void*)&(shc->hc2) );
FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc3), sizeof(shc->hc3)) == NULL, "hc3 (%p) failed init", (void*)&(shc->hc3) );
FUZ_CHECKTEST( LZ4_initStreamHC((char*)&(shc->hc1) + 1, sizeof(shc->hc1)) != NULL,
"hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->hc1) + 1) );
free(shc);
}
DISPLAYLEVEL(3, "all inits OK \n");
/* LZ4 HC streaming tests */
{ LZ4_streamHC_t sHC; /* statically allocated */
int result;
LZ4_initStreamHC(&sHC, sizeof(sHC));
/* Allocation test */
DISPLAYLEVEL(3, "Basic HC allocation : ");
{ LZ4_streamHC_t* const sp = LZ4_createStreamHC();
FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
LZ4_freeStreamHC(sp);
}
DISPLAYLEVEL(3, "OK \n");
/* simple HC compression test */
DISPLAYLEVEL(3, "Simple HC round-trip : ");
{ U64 const crc64 = XXH64(testInput, testCompressedSize, 0);
LZ4_setCompressionLevel(&sHC, compressionLevel);
result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
{ U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() decompression corruption");
} }
DISPLAYLEVEL(3, "OK \n");
/* deprecated streaming API */
DISPLAYLEVEL(3, "LZ4_compressHC2_continue() round-trip : ");
{ int const srcSize = 64 KB; /* LZ4_compressBound(srcSize) fits into testCompressed */
void* const ctx = LZ4_createHC(testInput);
FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
result = LZ4_compressHC2_continue(ctx, testInput, testCompressed, srcSize, compressionLevel);
FUZ_CHECKTEST(result<=0, "LZ4_compressHC2_continue() compression failed");
result = LZ4_decompress_safe(testCompressed, testVerify, result, srcSize);
FUZ_CHECKTEST(result!=srcSize || memcmp(testInput, testVerify, (size_t)srcSize),
"LZ4_compressHC2_continue() : corrupted round-trip");
LZ4_freeHC(ctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "level 2 : the match tested at ip+1 must not be ip itself, inside runs : ");
{ char runs[2 + 4096]; /* 0xFF 0x00, then runs of 10 x 0xFF and 10 x 0x00 */
int i;
runs[0] = (char)0xFF; runs[1] = 0;
for (i = 0; i < 4096; i++) runs[2 + i] = (i % 20) < 10 ? (char)0xFF : 0;
result = LZ4_compress_HC(runs, testCompressed, (int)sizeof(runs), (int)testCompressedSize, 2);
/* ~1.6 KB (one literal and one offset 1 per run), instead of < 50 bytes */
FUZ_CHECKTEST(result <= 0 || result > 100, "%i bytes", result);
}
DISPLAYLEVEL(3, "OK \n");
/* saveDictHC test #926 */
DISPLAYLEVEL(3, "saveDictHC test #926 : ");
{ LZ4_streamHC_t* const ctx = LZ4_initStreamHC(&sHC, sizeof(sHC));
assert(ctx != NULL); /* ensure init is successful */
/* Check access violation with asan */
FUZ_CHECKTEST( LZ4_saveDictHC(ctx, NULL, 0) != 0,
"LZ4_saveDictHC() can't save anything into (NULL,0)");
/* Check access violation with asan */
{ char tmp_buffer[240] = { 0 };
FUZ_CHECKTEST( LZ4_saveDictHC(ctx, tmp_buffer, sizeof(tmp_buffer)) != 0,
"LZ4_saveDictHC() can't save anything since compression hasn't started");
}
/* next block is compressed as the first one of the stream,
* without (attach==0) or with (attach==1) an attached dictionary */
{ LZ4_streamHC_t* const dictHC = LZ4_createStreamHC();
int const levels[] = { LZ4HC_CLEVEL_DEFAULT, 3, LZ4HC_CLEVEL_OPT_MIN, LZ4HC_CLEVEL_MAX }; /* first : level of LZ4_initStreamHC() */
int attach, l;
FUZ_CHECKTEST(dictHC == NULL, "LZ4_createStreamHC() allocation failed");
for (l = 0; l < (int)(sizeof(levels) / sizeof(levels[0])); l++) {
for (attach = 0; attach <= 1; attach++) {
char saveBuffer[240];
int cSize, dSize;
if (l || attach) LZ4_resetStreamHC_fast(ctx, levels[l]); /* first test : right after init */
if (attach) {
LZ4_resetStreamHC_fast(dictHC, levels[l]);
FUZ_CHECKTEST(LZ4_loadDictHC(dictHC, testInput, 64 KB) != 64 KB, "LZ4_loadDictHC() failed");
LZ4_attach_HC_dictionary(ctx, dictHC);
}
FUZ_CHECKTEST( LZ4_saveDictHC(ctx, saveBuffer, sizeof(saveBuffer)) != 0,
"LZ4_saveDictHC() can't save anything since compression hasn't started");
cSize = LZ4_compress_HC_continue(ctx, testInput + 64 KB, testCompressed, 64 KB, testCompressedSize);
FUZ_CHECKTEST(cSize <= 0, "LZ4_compress_HC_continue() failed");
dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, 64 KB,
testInput, attach ? 64 KB : 0);
FUZ_CHECKTEST(dSize != 64 KB || memcmp(testVerify, testInput + 64 KB, 64 KB),
"level %i, dictionary %i : block compressed after LZ4_saveDictHC() is corrupted", levels[l], attach);
} }
LZ4_freeStreamHC(dictHC);
} }
DISPLAYLEVEL(3, "OK \n");
/* long sequence test */
DISPLAYLEVEL(3, "Long sequence HC_destSize test : ");
{ size_t const blockSize = 1 MB;
size_t const targetSize = 4116; /* size carefully selected to trigger an overflow */
void* const block = malloc(blockSize);
void* const dstBlock = malloc(targetSize+1);
BYTE const sentinel = 101;
int srcSize;
assert(block != NULL); assert(dstBlock != NULL);
memset(block, 0, blockSize);
((char*)dstBlock)[targetSize] = sentinel;
LZ4_resetStreamHC_fast(&sHC, 3);
assert(blockSize < INT_MAX);
srcSize = (int)blockSize;
assert(targetSize < INT_MAX);
result = LZ4_compress_HC_destSize(&sHC, (const char*)block, (char*)dstBlock, &srcSize, (int)targetSize, 3);
DISPLAYLEVEL(4, "cSize=%i; readSize=%i; ", result, srcSize);
FUZ_CHECKTEST(result != 4116, "LZ4_compress_HC_destSize() : "
"compression (%i->%i) must fill dstBuffer (%i) exactly",
srcSize, result, (int)targetSize);
FUZ_CHECKTEST(((char*)dstBlock)[targetSize] != sentinel,
"LZ4_compress_HC_destSize() overwrites dst buffer");
FUZ_CHECKTEST(srcSize < 1045000, "LZ4_compress_HC_destSize() doesn't compress enough"
" (%i -> %i , expected > %i)", srcSize, result, 1045000);
LZ4_resetStreamHC_fast(&sHC, 3); /* make sure the context is clean after the test */
free(block);
free(dstBlock);
}
DISPLAYLEVEL(3, " OK \n");
/* a failed compression marks the state as dirty, which is fully re-initialized at next reset */
DISPLAYLEVEL(3, "favorDecSpeed survives a failed compression : ");
{ int const srcSize = 64 KB;
int const level = LZ4HC_CLEVEL_OPT_MIN;
int refSize, r;
U32 refCrc;
/* reference : favor decompression speed */
LZ4_initStreamHC(&sHC, sizeof(sHC));
LZ4_resetStreamHC_fast(&sHC, level);
LZ4_favorDecompressionSpeed(&sHC, 1);
refSize = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, srcSize, testCompressedSize);
FUZ_CHECKTEST(refSize <= 0, "LZ4_compress_HC_continue() failed");
refCrc = XXH32(testCompressed, (size_t)refSize, 0);
/* this test is only meaningful if favorDecSpeed changes the output */
LZ4_resetStreamHC_fast(&sHC, level);
LZ4_favorDecompressionSpeed(&sHC, 0);
r = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, srcSize, testCompressedSize);
FUZ_CHECKTEST(r == refSize && XXH32(testCompressed, (size_t)r, 0) == refCrc,
"favorDecSpeed doesn't change the output of this test input");
/* failed compression */
LZ4_resetStreamHC_fast(&sHC, level);
LZ4_favorDecompressionSpeed(&sHC, 1);
r = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, srcSize, refSize - 1);
FUZ_CHECKTEST(r != 0, "compression into a too small buffer should fail");
/* next stream : must still favor decompression speed */
LZ4_resetStreamHC_fast(&sHC, level);
r = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, srcSize, testCompressedSize);
FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc,
"favorDecSpeed lost after a failed compression (%i vs %i bytes)", r, refSize);
}
DISPLAYLEVEL(3, "OK \n");
/* simple dictionary HC compression test */
DISPLAYLEVEL(3, "HC dictionary compression test : ");
{ U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC_fast(&sHC, compressionLevel);
LZ4_loadDictHC(&sHC, testInput, 64 KB);
{ int const cSize = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(cSize==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : @return = %i", cSize);
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
{ int const dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, testCompressedSize, testInput, 64 KB);
FUZ_CHECKTEST(dSize!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed");
} }
{ U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption");
} }
DISPLAYLEVEL(3, " OK \n");
/* multiple HC compression test with dictionary */
{ int result1, result2;
int segSize = testCompressedSize / 2;
XXH64_hash_t const crc64 = ( (void)assert((unsigned)segSize + testCompressedSize < testInputSize) ,
XXH64(testInput + segSize, testCompressedSize, 0) );
LZ4_resetStreamHC_fast(&sHC, compressionLevel);
LZ4_loadDictHC(&sHC, testInput, segSize);
result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
FUZ_CHECKTEST(result1==0, "LZ4_compress_HC_continue() dictionary compression failed : result = %i", result1);
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*(size_t)segSize, testCompressed+result1, segSize, segSize-1);
FUZ_CHECKTEST(result2==0, "LZ4_compress_HC_continue() dictionary compression failed : result = %i", result2);
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize);
FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed");
result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize);
FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed");
{ XXH64_hash_t const crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
} }
/* remote dictionary HC compression test */
{ U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC_fast(&sHC, compressionLevel);
LZ4_loadDictHC(&sHC, testInput, 32 KB);
result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
FUZ_CHECKTEST(result==0, "LZ4_compress_HC_continue() remote dictionary failed : result = %i", result);
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB);
FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test");
{ U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption");
} }
/* multiple HC compression with ext. dictionary */
{ XXH64_state_t crcOrigState;
XXH64_state_t crcNewState;
@@ -2222,85 +1599,6 @@ static void FUZ_unitTests(int compressionLevel)
DISPLAYLEVEL(3, "OK \n");
}
DISPLAYLEVEL(3, "LZ4_compress_HC_destSize : ");
/* encode congenerical sequence test for HC compressors */
{ LZ4_streamHC_t* const sHC = LZ4_createStreamHC();
int const src_buf_size = 3 MB;
int const dst_buf_size = 6 KB;
int const payload = 0;
int const dst_step = 43;
int const dst_min_len = 33 + (FUZ_rand(&randState) % dst_step);
int const dst_max_len = 5000;
int slen, dlen;
char* sbuf1 = (char*)malloc(src_buf_size + 1);
char* sbuf2 = (char*)malloc(src_buf_size + 1);
char* dbuf1 = (char*)malloc(dst_buf_size + 1);
char* dbuf2 = (char*)malloc(dst_buf_size + 1);
assert(sHC != NULL);
assert(dst_buf_size > dst_max_len);
if (!sbuf1 || !sbuf2 || !dbuf1 || !dbuf2) {
EXIT_MSG("not enough memory for FUZ_unitTests (destSize)");
}
for (dlen = dst_min_len; dlen <= dst_max_len; dlen += dst_step) {
int src_len = (dlen - 10)*255 + 24;
if (src_len + 10 >= src_buf_size) break; /* END of check */
for (slen = src_len - 3; slen <= src_len + 3; slen++) {
int srcsz1, srcsz2;
int dsz1, dsz2;
int res1, res2;
char const endchk = (char)0x88;
DISPLAYLEVEL(5, "slen = %i, ", slen);
srcsz1 = slen;
memset(sbuf1, payload, slen);
memset(dbuf1, 0, dlen);
dbuf1[dlen] = endchk;
dsz1 = LZ4_compress_destSize(sbuf1, dbuf1, &srcsz1, dlen);
DISPLAYLEVEL(5, "LZ4_compress_destSize: %i bytes compressed into %i bytes, ", srcsz1, dsz1);
DISPLAYLEVEL(5, "last token : 0x%0X, ", dbuf1[dsz1 - 6]);
DISPLAYLEVEL(5, "last ML extra lenbyte : 0x%0X, \n", dbuf1[dsz1 - 7]);
FUZ_CHECKTEST(dbuf1[dlen] != endchk, "LZ4_compress_destSize() overwrite dst buffer !");
FUZ_CHECKTEST(dsz1 <= 0, "LZ4_compress_destSize() compression failed");
FUZ_CHECKTEST(dsz1 > dlen, "LZ4_compress_destSize() result larger than dst buffer !");
FUZ_CHECKTEST(srcsz1 > slen, "LZ4_compress_destSize() read more than src buffer !");
res1 = LZ4_decompress_safe(dbuf1, sbuf1, dsz1, src_buf_size);
FUZ_CHECKTEST(res1 != srcsz1, "LZ4_compress_destSize() decompression failed!");
srcsz2 = slen;
memset(sbuf2, payload, slen);
memset(dbuf2, 0, dlen);
dbuf2[dlen] = endchk;
LZ4_resetStreamHC(sHC, compressionLevel);
dsz2 = LZ4_compress_HC_destSize(sHC, sbuf2, dbuf2, &srcsz2, dlen, compressionLevel);
DISPLAYLEVEL(5, "LZ4_compress_HC_destSize: %i bytes compressed into %i bytes, ", srcsz2, dsz2);
DISPLAYLEVEL(5, "last token : 0x%0X, ", dbuf2[dsz2 - 6]);
DISPLAYLEVEL(5, "last ML extra lenbyte : 0x%0X, \n", dbuf2[dsz2 - 7]);
FUZ_CHECKTEST(dbuf2[dlen] != endchk, "LZ4_compress_HC_destSize() overwrite dst buffer !");
FUZ_CHECKTEST(dsz2 <= 0, "LZ4_compress_HC_destSize() compression failed");
FUZ_CHECKTEST(dsz2 > dlen, "LZ4_compress_HC_destSize() result larger than dst buffer !");
FUZ_CHECKTEST(srcsz2 > slen, "LZ4_compress_HC_destSize() read more than src buffer !");
FUZ_CHECKTEST(dsz2 != dsz1, "LZ4_compress_HC_destSize() return incorrect result !");
FUZ_CHECKTEST(srcsz2 != srcsz1, "LZ4_compress_HC_destSize() return incorrect src buffer size "
": srcsz2(%i) != srcsz1(%i)", srcsz2, srcsz1);
FUZ_CHECKTEST(memcmp(dbuf2, dbuf1, (size_t)dsz2), "LZ4_compress_HC_destSize() return incorrect data into dst buffer !");
res2 = LZ4_decompress_safe(dbuf2, sbuf1, dsz2, src_buf_size);
FUZ_CHECKTEST(res2 != srcsz1, "LZ4_compress_HC_destSize() decompression failed!");
FUZ_CHECKTEST(memcmp(sbuf1, sbuf2, (size_t)res2), "LZ4_compress_HC_destSize() decompression corruption!");
}
}
LZ4_freeStreamHC(sHC);
free(sbuf1);
free(sbuf2);
free(dbuf1);
free(dbuf2);
}
DISPLAYLEVEL(3, " OK \n");
/* clean up */
free(testInput);
free(testCompressed);

12
tests/unittest.c
View File

@@ -24,6 +24,7 @@
*/
#include "unittest.h"
#include "datagen.h" /* RDG_genBuffer */
#include <setjmp.h> /* setjmp, longjmp */
#include <stdarg.h> /* va_list */
#include <stdio.h> /* printf, snprintf */
@@ -35,9 +36,13 @@
/*-************************************
* Test suites : one per test file
**************************************/
extern const UT_suite UT_suite_lz4;
extern const UT_suite UT_suite_lz4hc;
extern const UT_suite UT_suite_lz4frame;
static const UT_suite* const g_suites[] = {
&UT_suite_lz4,
&UT_suite_lz4hc,
&UT_suite_lz4frame,
};
@@ -80,6 +85,13 @@ unsigned UT_rand(void)
return g_randState;
}
void* UT_sample(size_t size)
{
void* const buffer = UT_malloc(size);
RDG_genBuffer(buffer, size, 0.5, 0.0, UT_rand());
return buffer;
}
/*-************************************
* Assertions

12
tests/unittest.h
View File

@@ -66,6 +66,14 @@
#endif
/*-************************************
* Common definitions
**************************************/
#define KB *(1U<<10)
#define MB *(1U<<20)
#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
/*-************************************
* Tests
**************************************/
@@ -111,6 +119,10 @@ void* UT_defer(void (*release)(void*), void* ptr);
* pseudo-random generator, restarted from the same seed (-s#) for each test */
unsigned UT_rand(void);
/* UT_sample() :
* @return : @size bytes of compressible data, generated from UT_rand(), released at the end of the test */
void* UT_sample(size_t size);
/*-************************************
* Implementation of assertions

425
tests/unittest_lz4.c Normal file
View File

@@ -0,0 +1,425 @@
/*
unittest_lz4 - unit tests of lz4 (block format, fast levels)
Copyright (C) Yann Collet 2026
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 homepage : http://www.lz4.org
- LZ4 source repository : https://github.com/lz4/lz4
*/
#include "unittest.h"
#include <limits.h> /* INT_MIN, INT_MAX */
#include <string.h> /* memset, memcpy */
#define LZ4_STATIC_LINKING_ONLY /* LZ4_compress_destSize_extState, LZ4_attach_dictionary */
#include "lz4.h"
#include "lz4hc.h"
/*-************************************
* Helpers
**************************************/
static void freeStream(void* stream) { LZ4_freeStream((LZ4_stream_t*)stream); }
static void freeStreamHC(void* stream) { LZ4_freeStreamHC((LZ4_streamHC_t*)stream); }
/* initialized stream, released at the end of the test */
static LZ4_stream_t* newStream(void)
{
LZ4_stream_t* const stream = (LZ4_stream_t*)UT_defer(freeStream, LZ4_createStream());
ASSERT(stream != NULL);
return stream;
}
/* 64 KB block, beyond the table clearing threshold whatever LZ4_MEMORY_USAGE,
* in which the first 4 bytes reappear with a different 5th byte :
* a match only found when starting from a cleared table */
#define BLOCK_64K (64 KB)
static char* clearedTableBlock(void)
{
char* const block = (char*)UT_sample(BLOCK_64K);
memcpy(block, "ABCDx", 5);
memcpy(block + 16, "ABCDy", 5);
return block;
}
/* large enough to enter any decoding loop */
#define DECODED_CAPACITY (64 KB)
/*-************************************
* Decoding of invalid input
**************************************/
TEST(decompress_safe_emptyInput)
{
char const src[1] = { 0 };
char* const decoded = (char*)UT_malloc(DECODED_CAPACITY);
ASSERT(LZ4_decompress_safe(src, decoded, 0, DECODED_CAPACITY) < 0);
}
TEST(decompress_safe_oneByteInput)
{
char const src[1] = { (char)0xFF };
char* const decoded = (char*)UT_malloc(DECODED_CAPACITY);
ASSERT(LZ4_decompress_safe(src, decoded, 1, DECODED_CAPACITY) < 0);
}
/* 14 bytes of literals, followed by a 14 byte match :
* must not read beyond the end of the input (#508) */
TEST(decompress_safe_shortcutEdgeCase)
{
char block[17];
char* const decoded = (char*)UT_malloc(DECODED_CAPACITY);
block[0] = (char)0xEE;
memset(block + 1, 0, 14);
block[15] = 14;
block[16] = 0;
ASSERT(LZ4_decompress_safe(block, decoded, sizeof(block), DECODED_CAPACITY) < 0);
}
/* out-of-bounds read regression test */
TEST(decompress_safe_all0xFF)
{
char block[31];
char* const decoded = (char*)UT_malloc(DECODED_CAPACITY);
memset(block, 0xFF, sizeof(block));
ASSERT(LZ4_decompress_safe(block, decoded, sizeof(block), DECODED_CAPACITY) < 0);
}
/* ip must not go beyond iend.
* The block has 14 literals, ML_MASK match token, offset=14, one extension byte=0,
* which positions ip exactly at iend when read_variable_length() reads the extension.
* - 65 bytes output : large enough to enter the fast loop (>=64),
* small enough that op+length >= oend-64 triggers the safe_match_copy path,
* which would loop back with ip==iend, triggering assert(ip<iend).
* - 40 bytes output : small enough to skip the fast loop (<64), large enough for the shortcut (>=32),
* which reads literals+offset, landing ip at iend-1, then goto _copy_match
* calls read_variable_length() with ip >= ilimit. */
TEST(decompress_safe_ipAtIend)
{
static const int dstCapacities[] = { 65, 40 };
char block[18];
char decoded[65];
size_t n;
block[0] = (char)0xEF; /* 14 literals, ML_MASK match */
memset(block + 1, 'A', 14); /* 14 literal bytes */
block[15] = 0x0E; block[16] = 0x00; /* offset = 14 */
block[17] = 0x00; /* match length extension = 0 */
for (n = 0; n < ARRAY_SIZE(dstCapacities); n++)
ASSERT(LZ4_decompress_safe(block, decoded, sizeof(block), dstCapacities[n]) < 0);
}
/* Invalid offset 0 must not leave previous output buffer content in decoded data
* (see doc/lz4_Block_format.md). 1 literal, then a match at offset 0,
* partially decoded into a 20-byte target which ends within the match. */
TEST(decompress_safe_partial_offset0)
{
char block[32];
char decoded[20];
int r, i;
memset(block, 0, sizeof(block));
block[0] = 0x1F; /* 1 literal, ML_MASK match */
block[1] = 'A'; /* literal */
block[2] = 0x00; block[3] = 0x00; /* offset = 0 (invalid) */
block[4] = 64; /* match length extension */
memset(decoded, 0xAA, sizeof(decoded));
r = LZ4_decompress_safe_partial(block, decoded, sizeof(block), sizeof(decoded), sizeof(decoded));
for (i = 1; i < r; i++)
ASSERT_MSG(decoded[i] != (char)0xAA, "previous content kept at position %i", i);
}
/*-************************************
* One-shot compression
**************************************/
/* to be tested with UBSan */
TEST(compress_default_nullInput)
{
char dst[16];
int const cSize = LZ4_compress_default(NULL, dst, 0, LZ4_compressBound(0));
ASSERT_EQ(cSize, 1);
ASSERT_EQ(dst[0], 0);
}
TEST(compress_default_nullInputAndOutput)
{
ASSERT_EQ(LZ4_compress_default(NULL, NULL, 0, 0), 0);
}
/* acceleration < 1 is replaced by the default acceleration, 1 */
TEST(compress_destSize_extState_outOfRangeAcceleration)
{
static const int accels[] = { 0, -1, -100, INT_MIN, INT_MAX };
int const srcSize = 128 KB;
int const targetDstSize = srcSize / 4; /* < LZ4_compressBound(srcSize) : fillOutput mode */
const char* const src = (const char*)UT_sample((size_t)srcSize);
char* const ref = (char*)UT_malloc((size_t)targetDstSize);
char* const dst = (char*)UT_malloc((size_t)targetDstSize);
char* const decoded = (char*)UT_malloc((size_t)srcSize);
LZ4_stream_t state;
int refConsumed = srcSize, refSize;
size_t n;
refSize = LZ4_compress_destSize_extState(&state, src, ref, &refConsumed, targetDstSize, 1);
ASSERT(refSize > 0);
for (n = 0; n < ARRAY_SIZE(accels); n++) {
int consumed = srcSize;
int const cSize = LZ4_compress_destSize_extState(&state, src, dst, &consumed, targetDstSize, accels[n]);
ASSERT_MSG(cSize > 0 && cSize <= targetDstSize, "acceleration %i : compression failed (%i)", accels[n], cSize);
if (accels[n] < 1) {
ASSERT_MSG(cSize == refSize && consumed == refConsumed && !memcmp(dst, ref, (size_t)cSize),
"acceleration %i : output differs from acceleration 1", accels[n]);
}
ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, srcSize), consumed);
ASSERT_MEMEQ(decoded, src, (size_t)consumed);
}
}
TEST(compress_default_inPlace)
{
int const srcSize = 65 KB;
int const maxCSize = LZ4_COMPRESSBOUND(srcSize);
int const bufferSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize);
const char* const src = (const char*)UT_sample((size_t)srcSize);
char* const buffer = (char*)UT_malloc((size_t)bufferSize);
char* const input = buffer + bufferSize - srcSize; /* at the end of the buffer */
char* const decoded = (char*)UT_malloc((size_t)srcSize);
int cSize;
memcpy(input, src, (size_t)srcSize);
cSize = LZ4_compress_default(input, buffer, srcSize, maxCSize);
ASSERT(cSize > 0);
ASSERT_EQ(LZ4_decompress_safe(buffer, decoded, cSize, srcSize), srcSize);
ASSERT_MEMEQ(decoded, src, (size_t)srcSize);
}
/*-************************************
* Streaming compression
**************************************/
TEST(initStream_alignment)
{
typedef struct {
LZ4_stream_t state1;
LZ4_stream_t state2;
char c;
LZ4_stream_t state3;
} shct;
shct* const shc = (shct*)UT_malloc(sizeof(shct));
memset(shc, 0, sizeof(*shc));
ASSERT(LZ4_initStream(&shc->state1, sizeof(shc->state1)) != NULL);
ASSERT(LZ4_initStream(&shc->state2, sizeof(shc->state2)) != NULL);
ASSERT(LZ4_initStream(&shc->state3, sizeof(shc->state3)) != NULL);
ASSERT(LZ4_initStream((char*)&shc->state1 + 1, sizeof(shc->state1)) == NULL); /* bad alignment */
}
TEST(createStream)
{
newStream();
}
TEST(compress_fast_continue_roundTrip)
{
int const srcSize = 130 KB;
const char* const src = (const char*)UT_sample((size_t)srcSize);
char* const dst = (char*)UT_malloc((size_t)srcSize);
char* const decoded = (char*)UT_malloc((size_t)srcSize);
int const cSize = LZ4_compress_fast_continue(newStream(), src, dst, srcSize, srcSize - 1, 1);
ASSERT(cSize > 0);
ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, srcSize), srcSize);
ASSERT_MEMEQ(decoded, src, (size_t)srcSize);
}
TEST(resetStream_fast_largeFirstBlock) /* a large first block gives the same output as a new stream */
{
const char* const block = clearedTableBlock();
int const capacity = LZ4_COMPRESSBOUND(BLOCK_64K);
char* const ref = (char*)UT_malloc((size_t)capacity);
char* const dst = (char*)UT_malloc((size_t)capacity);
char saveBuffer[1];
LZ4_stream_t* const stream = newStream();
int const refSize = LZ4_compress_fast_continue(stream, block, ref, BLOCK_64K, capacity, 1);
ASSERT(refSize > 0);
LZ4_resetStream_fast(stream); /* the state now has a history */
ASSERT_EQ(LZ4_compress_fast_continue(stream, block, dst, BLOCK_64K, capacity, 1), refSize);
ASSERT_MEMEQ(dst, ref, (size_t)refSize);
LZ4_saveDict(stream, saveBuffer, 0); /* drop history, without reset */
ASSERT_EQ(LZ4_compress_fast_continue(stream, block, dst, BLOCK_64K, capacity, 1), refSize);
ASSERT_MEMEQ(dst, ref, (size_t)refSize);
}
/* a dictionary below HASH_UNIT on all platforms is ignored by LZ4_loadDict() :
* a large first block gives the same output as a new stream */
TEST(loadDict_tooSmall)
{
const char* const block = clearedTableBlock();
const char* const dict = (const char*)UT_sample(4);
int const capacity = LZ4_COMPRESSBOUND(BLOCK_64K);
char* const ref = (char*)UT_malloc((size_t)capacity);
char* const dst = (char*)UT_malloc((size_t)capacity);
LZ4_stream_t* const stream = newStream();
LZ4_stream_t* const dictStream = newStream();
int const refSize = LZ4_compress_fast_continue(stream, block, ref, BLOCK_64K, capacity, 1);
int dictSize;
ASSERT(refSize > 0);
for (dictSize = 0; dictSize < 4; dictSize++) {
LZ4_loadDict(dictStream, dict, dictSize);
ASSERT_MSG(LZ4_compress_fast_continue(dictStream, block, dst, BLOCK_64K, capacity, 1) == refSize
&& !memcmp(dst, ref, (size_t)refSize),
"after LZ4_loadDict() (%i bytes) : output differs from a new stream", dictSize);
LZ4_loadDict(dictStream, dict, dictSize);
LZ4_initStream(stream, sizeof(*stream));
LZ4_attach_dictionary(stream, dictStream);
ASSERT_MSG(LZ4_compress_fast_continue(stream, block, dst, BLOCK_64K, capacity, 1) == refSize
&& !memcmp(dst, ref, (size_t)refSize),
"after LZ4_attach_dictionary() (%i bytes) : output differs from a new stream", dictSize);
}
}
TEST(saveDict_afterInit)
{
LZ4_stream_t* const stream = newStream();
char saveBuffer[240];
ASSERT_EQ(LZ4_saveDict(stream, NULL, 0), 0);
ASSERT_EQ(LZ4_saveDict(stream, saveBuffer, sizeof(saveBuffer)), 0); /* compression hasn't started */
}
/* a negative size saves nothing (instead of wrapping to 64 KB),
* and the next block no longer references the previous dictionary :
* it copies the end of the dictionary, so it would match it */
TEST(saveDict_negativeSize)
{
const char* const dict = (const char*)UT_sample(64 KB);
char* const block = (char*)UT_malloc(4 KB);
int const capacity = LZ4_COMPRESSBOUND(4 KB);
char* const dst = (char*)UT_malloc((size_t)capacity);
char* const decoded = (char*)UT_malloc(4 KB);
LZ4_stream_t* const stream = newStream();
char saveBuffer[16];
int cSize;
memcpy(block, dict + 60 KB, 4 KB);
ASSERT(LZ4_loadDict(stream, dict, 64 KB) > 0);
ASSERT_EQ(LZ4_saveDict(stream, saveBuffer, -1), 0);
cSize = LZ4_compress_fast_continue(stream, block, dst, 4 KB, capacity, 1);
ASSERT(cSize > 0);
ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, 4 KB), 4 KB); /* no reference to the dictionary */
ASSERT_MEMEQ(decoded, block, 4 KB);
}
/* chains of short matches must not hide the period (#1805) */
TEST(compress_periodicRuns)
{
static const char dict[8] = { 0, 0, 0, 0, 0, 0, 0, 4 };
char page[8 + 4096]; /* dict, then a 4 KB page alternating runs of 16 x 0xFF and 16 x 0x00 */
int const capacity = LZ4_COMPRESSBOUND(sizeof(page));
char* const dst = (char*)UT_malloc((size_t)capacity);
LZ4_stream_t* const stream = newStream();
LZ4_stream_t* const dictStream = newStream();
int r;
memcpy(page, dict, sizeof(dict));
for (r = 0; r < 32; r++) { /* several rotations compressed to ~1.5 KB, instead of < 60 bytes */
int i, c;
for (i = 0; i < 4096; i++)
page[8 + i] = (((i + r) % 4096 + 8) / 16) % 2 == 0 ? (char)0xFF : 0;
c = LZ4_compress_default(page, dst, (int)sizeof(page), capacity);
ASSERT_MSG(c > 0 && c <= 100, "rotation %i, dictionary bytes in the same block : %i bytes", r, c);
LZ4_initStream(stream, sizeof(*stream));
LZ4_loadDict(stream, dict, (int)sizeof(dict));
c = LZ4_compress_fast_continue(stream, page + 8, dst, 4096, capacity, 1);
ASSERT_MSG(c > 0 && c <= 100, "rotation %i, LZ4_loadDict() : %i bytes", r, c);
LZ4_initStream(dictStream, sizeof(*dictStream));
LZ4_loadDict(dictStream, dict, (int)sizeof(dict));
LZ4_initStream(stream, sizeof(*stream));
LZ4_attach_dictionary(stream, dictStream);
c = LZ4_compress_fast_continue(stream, page + 8, dst, 4096, capacity, 1);
ASSERT_MSG(c > 0 && c <= 100, "rotation %i, LZ4_attach_dictionary() : %i bytes", r, c);
}
}
/* the new block can overwrite the previous one (its dictionary) : any overlap must be handled */
TEST(compress_continue_inputOverwritesPreviousBlock)
{
/* block 1 : 1000 bytes at position 1000 ; block 2 : { position, size } */
static const int overlaps[4][2] = {
{ 500, 1000 }, /* covers the beginning of block 1 */
{ 1000, 1000 }, /* same place */
{ 500, 2000 }, /* covers block 1 entirely */
{ 1500, 1000 } }; /* covers the end of block 1 */
int const pos1 = 1000, size1 = 1000;
int const capacity = LZ4_COMPRESSBOUND(4 KB);
char* const ring = (char*)UT_malloc(4 KB);
char* const src = (char*)UT_malloc(4 KB); /* content of both blocks */
char* const cBlock1 = (char*)UT_malloc((size_t)capacity);
char* const cBlock2 = (char*)UT_malloc((size_t)capacity);
char* const decoded = (char*)UT_malloc(4 KB);
LZ4_stream_t* const stream = newStream();
LZ4_streamHC_t* const streamHC = (LZ4_streamHC_t*)UT_defer(freeStreamHC, LZ4_createStreamHC());
int ov, hc, trial;
ASSERT(streamHC != NULL);
memset(ring, 0, 4 KB);
for (ov = 0; ov < 4; ov++)
for (hc = 0; hc < 2; hc++)
for (trial = 0; trial < 64; trial++) {
int const pos2 = overlaps[ov][0], size2 = overlaps[ov][1];
LZ4_streamDecode_t sd;
int c1, c2, i, r;
for (i = 0; i < size1 + size2; i++) src[i] = "ACGT"[UT_rand() & 3]; /* many matches */
LZ4_initStream(stream, sizeof(*stream));
LZ4_initStreamHC(streamHC, sizeof(*streamHC));
memcpy(ring + pos1, src, (size_t)size1);
c1 = hc ? LZ4_compress_HC_continue(streamHC, ring + pos1, cBlock1, size1, capacity)
: LZ4_compress_fast_continue(stream, ring + pos1, cBlock1, size1, capacity, 1);
memcpy(ring + pos2, src + size1, (size_t)size2);
c2 = hc ? LZ4_compress_HC_continue(streamHC, ring + pos2, cBlock2, size2, capacity)
: LZ4_compress_fast_continue(stream, ring + pos2, cBlock2, size2, capacity, 1);
ASSERT(c1 > 0 && c2 > 0);
LZ4_setStreamDecode(&sd, NULL, 0);
r = LZ4_decompress_safe_continue(&sd, cBlock1, decoded, c1, size1);
r += LZ4_decompress_safe_continue(&sd, cBlock2, decoded + size1, c2, size2);
ASSERT_MSG(r == size1 + size2 && !memcmp(decoded, src, (size_t)(size1 + size2)),
"%s : corrupted output, block 2 (position %i, size %i) overwrites block 1 (position %i, size %i)",
hc ? "LZ4_compress_HC_continue()" : "LZ4_compress_fast_continue()", pos2, size2, pos1, size1);
}
}
static const UT_test tests[] = {
UT_TEST(decompress_safe_emptyInput),
UT_TEST(decompress_safe_oneByteInput),
UT_TEST(decompress_safe_shortcutEdgeCase),
UT_TEST(decompress_safe_all0xFF),
UT_TEST(decompress_safe_ipAtIend),
UT_TEST(decompress_safe_partial_offset0),
UT_TEST(compress_default_nullInput),
UT_TEST(compress_default_nullInputAndOutput),
UT_TEST(compress_destSize_extState_outOfRangeAcceleration),
UT_TEST(compress_default_inPlace),
UT_TEST(initStream_alignment),
UT_TEST(createStream),
UT_TEST(compress_fast_continue_roundTrip),
UT_TEST(resetStream_fast_largeFirstBlock),
UT_TEST(loadDict_tooSmall),
UT_TEST(saveDict_afterInit),
UT_TEST(saveDict_negativeSize),
UT_TEST(compress_periodicRuns),
UT_TEST(compress_continue_inputOverwritesPreviousBlock),
};
UT_SUITE(lz4, tests);

36
tests/unittest_lz4frame.c
View File

@@ -30,12 +30,6 @@
#include "lz4frame.h"
#define LZ4_STATIC_LINKING_ONLY /* LZ4_ACCELERATION_MAX */
#include "lz4.h"
#include "datagen.h" /* RDG_genBuffer */
#define KB *(1U<<10)
#define MB *(1U<<20)
#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
/*-************************************
* Helpers
@@ -59,14 +53,6 @@ static void checkLZ4FError(size_t r, LZ4F_errorCodes code,
LZ4F_isError(r) ? LZ4F_getErrorName(r) : "no error");
}
/* @return : @size bytes of compressible data, released at the end of the test */
static void* sample(size_t size)
{
void* const buffer = UT_malloc(size);
RDG_genBuffer(buffer, size, 0.5, 0.0, UT_rand());
return buffer;
}
static void freeCctx(void* cctx) { LZ4F_freeCompressionContext((LZ4F_cctx*)cctx); }
static void freeDctx(void* dctx) { LZ4F_freeDecompressionContext((LZ4F_dctx*)dctx); }
static void freeCDict(void* cdict) { LZ4F_freeCDict((LZ4F_CDict*)cdict); }
@@ -202,7 +188,7 @@ TEST(compressFrame_veryNegativeLevels)
{
static const int levels[] = { -LZ4_ACCELERATION_MAX /* cap - 1 */, INT_MIN + 1, INT_MIN };
size_t const srcSize = 2 MB;
const void* const src = sample(srcSize);
const void* const src = UT_sample(srcSize);
size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL);
void* const ref = UT_malloc(capacity);
void* const frame = UT_malloc(capacity);
@@ -220,7 +206,7 @@ TEST(compressFrame_veryNegativeLevels)
TEST(compressFrame_validBlockSizeIDs)
{
size_t const srcSize = 256 KB;
const void* const src = sample(srcSize);
const void* const src = UT_sample(srcSize);
size_t n;
for (n = 0; n < ARRAY_SIZE(g_validBlockSizeIDs); n++) {
LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES;
@@ -250,7 +236,7 @@ TEST(compressBegin_invalidBlockSizeIDs)
TEST(compressBegin_failurePreservesFrame)
{
size_t const srcSize = 256 KB;
const void* const src = sample(srcSize);
const void* const src = UT_sample(srcSize);
LZ4F_cctx* const cctx = newCctx();
LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES;
LZ4F_preferences_t invalidPrefs = LZ4F_INIT_PREFERENCES;
@@ -276,7 +262,7 @@ TEST(compressBegin_afterFailedStateAllocation)
LZ4F_CustomMem const failableMem = { failableMalloc, failableCalloc, failableFree, &failAlloc };
LZ4F_cctx* const cctx = (LZ4F_cctx*)UT_defer(freeCctx, LZ4F_createCompressionContext_advanced(failableMem, LZ4F_VERSION));
size_t const srcSize = 64 KB;
const void* const src = sample(srcSize);
const void* const src = UT_sample(srcSize);
size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL);
void* const frame = UT_malloc(capacity);
LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES;
@@ -294,7 +280,7 @@ TEST(compressBegin_afterFailedStateAllocation)
TEST(compressEnd_contentSize)
{
size_t const srcSize = 2 MB;
const void* const src = sample(srcSize);
const void* const src = UT_sample(srcSize);
LZ4F_cctx* const cctx = newCctx();
LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES;
size_t const capacity = LZ4F_compressFrameBound(srcSize, &prefs);
@@ -310,7 +296,7 @@ TEST(compressEnd_contentSize)
TEST(compressEnd_wrongContentSize)
{
size_t const srcSize = 2 MB;
const void* const src = sample(srcSize);
const void* const src = UT_sample(srcSize);
LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES;
size_t const capacity = LZ4F_compressFrameBound(srcSize, &prefs);
void* const frame = UT_malloc(capacity);
@@ -322,7 +308,7 @@ TEST(compressBegin_usingDict)
{
size_t const dictSize = 7 KB; /* small enough for LZ4_MEMORY_USAGE == 10 */
size_t const srcSize = 66 KB; /* must be > 64 KB to avoid short-size optimizations */
const char* const dict = (const char*)sample(dictSize + srcSize);
const char* const dict = (const char*)UT_sample(dictSize + srcSize);
const char* const src = dict + dictSize;
size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL); /* NULL preferences : 64 KB linked blocks */
void* const frame = UT_malloc(capacity);
@@ -337,7 +323,7 @@ TEST(createCDict_advanced)
int failAlloc = 1;
LZ4F_CustomMem const failableMem = { failableMalloc, failableCalloc, failableFree, &failAlloc };
size_t const dictSize = 7 KB;
const void* const dict = sample(dictSize);
const void* const dict = UT_sample(dictSize);
ASSERT(UT_defer(freeCDict, LZ4F_createCDict_advanced(failableMem, dict, dictSize)) == NULL); /* uses failableMem */
failAlloc = 0;
ASSERT(UT_defer(freeCDict, LZ4F_createCDict_advanced(failableMem, dict, dictSize)) != NULL);
@@ -355,7 +341,7 @@ static LZ4F_CDict* newCDict(const void* dict)
TEST(compressFrame_usingCDict)
{
size_t const srcSize = 65 KB; /* must be > 64 KB to avoid short-size optimizations */
const void* const src = sample(srcSize);
const void* const src = UT_sample(srcSize);
LZ4F_CDict* const cdict = newCDict(src);
size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL);
void* const frame = UT_malloc(capacity);
@@ -369,7 +355,7 @@ TEST(compressFrame_usingCDict_extremeLevels)
{
int const levels[] = { -3, LZ4F_compressionLevel_max() };
size_t const srcSize = CDICT_DICT_SIZE;
const void* const src = sample(srcSize);
const void* const src = UT_sample(srcSize);
LZ4F_CDict* const cdict = newCDict(src);
LZ4F_cctx* const cctx = newCctx();
size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL);
@@ -387,7 +373,7 @@ TEST(compressFrame_usingCDict_extremeLevels)
TEST(compressFrame_usingCDict_blockModes)
{
size_t const srcSize = 3 * 64 KB; /* a single block would always be independent */
const void* const src = sample(srcSize);
const void* const src = UT_sample(srcSize);
LZ4F_CDict* const cdict = newCDict(src);
LZ4F_cctx* const cctx = newCctx();
size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL);

374
tests/unittest_lz4hc.c Normal file
View File

@@ -0,0 +1,374 @@
/*
unittest_lz4hc - unit tests of lz4hc (block format, high compression levels)
Copyright (C) Yann Collet 2026
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 homepage : http://www.lz4.org
- LZ4 source repository : https://github.com/lz4/lz4
*/
#include "unittest.h"
#include <string.h> /* memset, memcpy, memcmp */
#define LZ4_DISABLE_DEPRECATE_WARNINGS /* LZ4_createHC, LZ4_compressHC2_continue */
#define LZ4_STATIC_LINKING_ONLY
#include "lz4.h"
#define LZ4_HC_STATIC_LINKING_ONLY /* LZ4_favorDecompressionSpeed */
#include "lz4hc.h"
/*-************************************
* Helpers
**************************************/
/* the default level, and the first level of the optimal parser */
static const int g_levels[] = { LZ4HC_CLEVEL_DEFAULT, LZ4HC_CLEVEL_OPT_MIN };
static void freeStreamHC(void* stream) { LZ4_freeStreamHC((LZ4_streamHC_t*)stream); }
/* stream, released at the end of the test */
static LZ4_streamHC_t* newStreamHC(int level)
{
LZ4_streamHC_t* const stream = (LZ4_streamHC_t*)UT_defer(freeStreamHC, LZ4_createStreamHC());
ASSERT(stream != NULL);
LZ4_resetStreamHC_fast(stream, level);
return stream;
}
/* a successful compression leaves the stream clean */
#define ASSERT_CLEAN(stream) ASSERT(!(stream)->internal_donotuse.dirty)
/*-************************************
* Tests
**************************************/
/* to be tested with UBSan */
TEST(compress_HC_nullInput)
{
int level;
for (level = LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++) {
char dst[16];
int const cSize = LZ4_compress_HC(NULL, dst, 0, LZ4_compressBound(0), level);
ASSERT_MSG(cSize == 1 && dst[0] == 0, "level %i : compressing empty input should give byte 0", level);
}
}
TEST(compress_HC_nullInputAndOutput)
{
int level;
for (level = LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++)
ASSERT_MSG(LZ4_compress_HC(NULL, NULL, 0, 0, level) == 0, "level %i : compressing into NULL must fail", level);
}
TEST(initStreamHC_alignment)
{
typedef struct {
LZ4_streamHC_t hc1;
LZ4_streamHC_t hc2;
char c;
LZ4_streamHC_t hc3;
} shct;
shct* const shc = (shct*)UT_malloc(sizeof(shct));
memset(shc, 0, sizeof(*shc));
ASSERT(LZ4_initStreamHC(&shc->hc1, sizeof(shc->hc1)) != NULL);
ASSERT(LZ4_initStreamHC(&shc->hc2, sizeof(shc->hc2)) != NULL);
ASSERT(LZ4_initStreamHC(&shc->hc3, sizeof(shc->hc3)) != NULL);
ASSERT(LZ4_initStreamHC((char*)&shc->hc1 + 1, sizeof(shc->hc1)) == NULL); /* bad alignment */
}
TEST(createStreamHC)
{
newStreamHC(LZ4HC_CLEVEL_DEFAULT);
}
TEST(compress_HC_continue_roundTrip)
{
int const srcSize = 130 KB;
const char* const src = (const char*)UT_sample((size_t)srcSize);
char* const dst = (char*)UT_malloc((size_t)srcSize);
char* const decoded = (char*)UT_malloc((size_t)srcSize);
LZ4_streamHC_t* const stream = (LZ4_streamHC_t*)UT_malloc(sizeof(LZ4_streamHC_t)); /* statically allocated */
size_t l;
for (l = 0; l < ARRAY_SIZE(g_levels); l++) {
int cSize;
LZ4_initStreamHC(stream, sizeof(*stream));
LZ4_setCompressionLevel(stream, g_levels[l]);
cSize = LZ4_compress_HC_continue(stream, src, dst, srcSize, srcSize - 1);
ASSERT(cSize > 0);
ASSERT_CLEAN(stream);
ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, srcSize), srcSize);
ASSERT_MEMEQ(decoded, src, (size_t)srcSize);
}
}
/* deprecated streaming API */
TEST(compressHC2_continue_roundTrip)
{
int const srcSize = 64 KB;
const char* const src = (const char*)UT_sample((size_t)srcSize);
char* const dst = (char*)UT_malloc((size_t)LZ4_compressBound(srcSize));
char* const decoded = (char*)UT_malloc((size_t)srcSize);
size_t l;
for (l = 0; l < ARRAY_SIZE(g_levels); l++) {
void* const ctx = LZ4_createHC(src);
int cSize;
ASSERT(ctx != NULL);
cSize = LZ4_compressHC2_continue(ctx, src, dst, srcSize, g_levels[l]);
LZ4_freeHC(ctx);
ASSERT(cSize > 0);
ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, srcSize), srcSize);
ASSERT_MEMEQ(decoded, src, (size_t)srcSize);
}
}
/* level 2 : the match tested at ip+1 must not be ip itself, inside runs */
TEST(compress_HC_level2_runs)
{
char runs[2 + 4096]; /* 0xFF 0x00, then runs of 10 x 0xFF and 10 x 0x00 */
int const capacity = LZ4_COMPRESSBOUND(sizeof(runs));
char* const dst = (char*)UT_malloc((size_t)capacity);
int i, cSize;
runs[0] = (char)0xFF; runs[1] = 0;
for (i = 0; i < 4096; i++) runs[2 + i] = (i % 20) < 10 ? (char)0xFF : 0;
cSize = LZ4_compress_HC(runs, dst, (int)sizeof(runs), capacity, 2);
/* ~1.6 KB (one literal and one offset 1 per run), instead of < 50 bytes */
ASSERT_MSG(cSize > 0 && cSize <= 100, "%i bytes", cSize);
}
/* #926 */
TEST(saveDictHC_beforeCompression)
{
/* first : level of LZ4_initStreamHC() */
static const int levels[] = { LZ4HC_CLEVEL_DEFAULT, 3, LZ4HC_CLEVEL_OPT_MIN, LZ4HC_CLEVEL_MAX };
const char* const dict = (const char*)UT_sample(128 KB);
const char* const src = dict + 64 KB;
int const capacity = LZ4_COMPRESSBOUND(64 KB);
char* const dst = (char*)UT_malloc((size_t)capacity);
char* const decoded = (char*)UT_malloc(64 KB);
LZ4_streamHC_t* const stream = (LZ4_streamHC_t*)UT_malloc(sizeof(LZ4_streamHC_t));
LZ4_streamHC_t* const dictStream = newStreamHC(LZ4HC_CLEVEL_DEFAULT);
char saveBuffer[240];
size_t l;
int attach;
ASSERT(LZ4_initStreamHC(stream, sizeof(*stream)) != NULL);
ASSERT_EQ(LZ4_saveDictHC(stream, NULL, 0), 0);
ASSERT_EQ(LZ4_saveDictHC(stream, saveBuffer, sizeof(saveBuffer)), 0); /* compression hasn't started */
/* next block is compressed as the first one of the stream,
* without (attach==0) or with (attach==1) an attached dictionary */
for (l = 0; l < ARRAY_SIZE(levels); l++) {
for (attach = 0; attach <= 1; attach++) {
int cSize;
if (l || attach) LZ4_resetStreamHC_fast(stream, levels[l]); /* first test : right after init */
if (attach) {
LZ4_resetStreamHC_fast(dictStream, levels[l]);
ASSERT_EQ(LZ4_loadDictHC(dictStream, dict, 64 KB), 64 KB);
LZ4_attach_HC_dictionary(stream, dictStream);
}
ASSERT_EQ(LZ4_saveDictHC(stream, saveBuffer, sizeof(saveBuffer)), 0);
cSize = LZ4_compress_HC_continue(stream, src, dst, 64 KB, capacity);
ASSERT(cSize > 0);
ASSERT_MSG(LZ4_decompress_safe_usingDict(dst, decoded, cSize, 64 KB, dict, attach ? 64 KB : 0) == 64 KB
&& !memcmp(decoded, src, 64 KB),
"level %i, dictionary %i : block compressed after LZ4_saveDictHC() is corrupted", levels[l], attach);
} }
}
TEST(compress_HC_destSize_longSequence)
{
size_t const srcSize = 1 MB;
size_t const targetSize = 4116; /* size carefully selected to trigger an overflow */
char* const src = (char*)UT_malloc(srcSize);
char* const dst = (char*)UT_malloc(targetSize + 1);
char const sentinel = 101;
LZ4_streamHC_t* const stream = newStreamHC(3);
int readSize = (int)srcSize;
int cSize;
memset(src, 0, srcSize);
dst[targetSize] = sentinel;
cSize = LZ4_compress_HC_destSize(stream, src, dst, &readSize, (int)targetSize, 3);
ASSERT_EQ(cSize, targetSize); /* fills dst exactly */
ASSERT_EQ(dst[targetSize], sentinel);
ASSERT_MSG(readSize >= 1045000, "doesn't compress enough (%i -> %i)", readSize, cSize);
}
/* a failed compression marks the state as dirty, which is fully re-initialized at next reset */
TEST(favorDecompressionSpeed_afterFailedCompression)
{
int const srcSize = 64 KB;
int const capacity = LZ4_COMPRESSBOUND(srcSize);
int const level = LZ4HC_CLEVEL_OPT_MIN;
const char* const src = (const char*)UT_sample((size_t)srcSize);
char* const ref = (char*)UT_malloc((size_t)capacity);
char* const dst = (char*)UT_malloc((size_t)capacity);
LZ4_streamHC_t* const stream = newStreamHC(level);
int refSize, cSize;
/* reference : favor decompression speed */
LZ4_favorDecompressionSpeed(stream, 1);
refSize = LZ4_compress_HC_continue(stream, src, ref, srcSize, capacity);
ASSERT(refSize > 0);
/* this test is only meaningful if favorDecSpeed changes the output */
LZ4_resetStreamHC_fast(stream, level);
LZ4_favorDecompressionSpeed(stream, 0);
cSize = LZ4_compress_HC_continue(stream, src, dst, srcSize, capacity);
ASSERT(cSize != refSize || memcmp(dst, ref, (size_t)cSize));
/* failed compression */
LZ4_resetStreamHC_fast(stream, level);
LZ4_favorDecompressionSpeed(stream, 1);
ASSERT_EQ(LZ4_compress_HC_continue(stream, src, dst, srcSize, refSize - 1), 0);
/* next stream : must still favor decompression speed */
LZ4_resetStreamHC_fast(stream, level);
ASSERT_EQ(LZ4_compress_HC_continue(stream, src, dst, srcSize, capacity), refSize);
ASSERT_MEMEQ(dst, ref, (size_t)refSize);
}
TEST(compress_HC_continue_dictionary)
{
int const dictSize = 64 KB;
int const srcSize = 130 KB;
const char* const dict = (const char*)UT_sample((size_t)(dictSize + srcSize));
const char* const src = dict + dictSize;
char* const dst = (char*)UT_malloc((size_t)srcSize);
char* const decoded = (char*)UT_malloc((size_t)srcSize);
size_t l;
for (l = 0; l < ARRAY_SIZE(g_levels); l++) {
LZ4_streamHC_t* const stream = newStreamHC(g_levels[l]);
int cSize;
LZ4_loadDictHC(stream, dict, dictSize);
cSize = LZ4_compress_HC_continue(stream, src, dst, srcSize, srcSize - 1);
ASSERT(cSize > 0);
ASSERT_CLEAN(stream);
ASSERT_EQ(LZ4_decompress_safe_usingDict(dst, decoded, cSize, srcSize, dict, dictSize), srcSize);
ASSERT_MEMEQ(decoded, src, (size_t)srcSize);
}
}
/* 2 blocks : the 2nd one also references the 1st one */
TEST(compress_HC_continue_dictionary_twoBlocks)
{
int const segSize = 65 KB;
const char* const input = (const char*)UT_sample((size_t)(3 * segSize)); /* dictionary, block 1, block 2 */
char* const dst = (char*)UT_malloc((size_t)(2 * segSize));
char* const decoded = (char*)UT_malloc((size_t)(2 * segSize));
size_t l;
for (l = 0; l < ARRAY_SIZE(g_levels); l++) {
LZ4_streamHC_t* const stream = newStreamHC(g_levels[l]);
int c1, c2;
LZ4_loadDictHC(stream, input, segSize);
c1 = LZ4_compress_HC_continue(stream, input + segSize, dst, segSize, segSize - 1);
ASSERT(c1 > 0);
ASSERT_CLEAN(stream);
c2 = LZ4_compress_HC_continue(stream, input + 2 * segSize, dst + c1, segSize, segSize - 1);
ASSERT(c2 > 0);
ASSERT_CLEAN(stream);
ASSERT_EQ(LZ4_decompress_safe_usingDict(dst, decoded, c1, segSize, input, segSize), segSize);
ASSERT_EQ(LZ4_decompress_safe_usingDict(dst + c1, decoded + segSize, c2, segSize, input, 2 * segSize), segSize);
ASSERT_MEMEQ(decoded, input + segSize, (size_t)(2 * segSize));
}
}
/* the dictionary doesn't immediately precede the input */
TEST(compress_HC_continue_remoteDictionary)
{
int const dictSize = 32 KB;
int const srcSize = 130 KB;
const char* const dict = (const char*)UT_sample((size_t)(64 KB + srcSize));
const char* const src = dict + 64 KB;
char* const dst = (char*)UT_malloc((size_t)srcSize);
char* const decoded = (char*)UT_malloc((size_t)srcSize);
size_t l;
for (l = 0; l < ARRAY_SIZE(g_levels); l++) {
LZ4_streamHC_t* const stream = newStreamHC(g_levels[l]);
int cSize;
LZ4_loadDictHC(stream, dict, dictSize);
cSize = LZ4_compress_HC_continue(stream, src, dst, srcSize, srcSize - 1);
ASSERT(cSize > 0);
ASSERT_CLEAN(stream);
ASSERT_EQ(LZ4_decompress_safe_usingDict(dst, decoded, cSize, srcSize, dict, dictSize), srcSize);
ASSERT_MEMEQ(decoded, src, (size_t)srcSize);
}
}
/* on long sequences of zeros, LZ4_compress_HC_destSize() and LZ4_compress_destSize() produce the same output */
TEST(compress_HC_destSize_sameAsFast)
{
int const srcCapacity = 3 MB;
int const dstCapacity = 6 KB;
int const dstStep = 43;
int const dstMinLen = 33 + (int)(UT_rand() % (unsigned)dstStep);
int const dstMaxLen = 5000;
char const endMark = (char)0x88;
char* const src1 = (char*)UT_malloc((size_t)srcCapacity + 1);
char* const src2 = (char*)UT_malloc((size_t)srcCapacity + 1);
char* const dst1 = (char*)UT_malloc((size_t)dstCapacity + 1);
char* const dst2 = (char*)UT_malloc((size_t)dstCapacity + 1);
LZ4_streamHC_t* const stream = newStreamHC(LZ4HC_CLEVEL_DEFAULT);
size_t l;
int dlen, slen;
ASSERT(dstCapacity > dstMaxLen);
for (l = 0; l < ARRAY_SIZE(g_levels); l++)
for (dlen = dstMinLen; dlen <= dstMaxLen; dlen += dstStep) {
int const srcLen = (dlen - 10) * 255 + 24;
if (srcLen + 10 >= srcCapacity) break;
for (slen = srcLen - 3; slen <= srcLen + 3; slen++) {
int srcSize1 = slen, srcSize2 = slen;
int dSize1, dSize2;
memset(src1, 0, (size_t)slen);
memset(dst1, 0, (size_t)dlen);
dst1[dlen] = endMark;
dSize1 = LZ4_compress_destSize(src1, dst1, &srcSize1, dlen);
ASSERT_EQ(dst1[dlen], endMark); /* no overwrite beyond dst */
ASSERT(dSize1 > 0 && dSize1 <= dlen);
ASSERT(srcSize1 <= slen);
ASSERT_EQ(LZ4_decompress_safe(dst1, src1, dSize1, srcCapacity), srcSize1);
memset(src2, 0, (size_t)slen);
memset(dst2, 0, (size_t)dlen);
dst2[dlen] = endMark;
LZ4_resetStreamHC(stream, g_levels[l]);
dSize2 = LZ4_compress_HC_destSize(stream, src2, dst2, &srcSize2, dlen, g_levels[l]);
ASSERT_EQ(dst2[dlen], endMark);
ASSERT_EQ(dSize2, dSize1);
ASSERT_EQ(srcSize2, srcSize1);
ASSERT_MEMEQ(dst2, dst1, (size_t)dSize2);
ASSERT_EQ(LZ4_decompress_safe(dst2, src1, dSize2, srcCapacity), srcSize1);
ASSERT_MEMEQ(src1, src2, (size_t)srcSize1);
} }
}
static const UT_test tests[] = {
UT_TEST(compress_HC_nullInput),
UT_TEST(compress_HC_nullInputAndOutput),
UT_TEST(initStreamHC_alignment),
UT_TEST(createStreamHC),
UT_TEST(compress_HC_continue_roundTrip),
UT_TEST(compressHC2_continue_roundTrip),
UT_TEST(compress_HC_level2_runs),
UT_TEST(saveDictHC_beforeCompression),
UT_TEST(compress_HC_destSize_longSequence),
UT_TEST(favorDecompressionSpeed_afterFailedCompression),
UT_TEST(compress_HC_continue_dictionary),
UT_TEST(compress_HC_continue_dictionary_twoBlocks),
UT_TEST(compress_HC_continue_remoteDictionary),
UT_TEST(compress_HC_destSize_sameAsFast),
};
UT_SUITE(lz4hc, tests);