From 5d63c34ac070fb3f8cebfba07eb2ca5fe1137d61 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Oct 2026 17:34:15 -0700 Subject: [PATCH] tests: move the simple block unit tests from fuzzer.c to unittest 33 tests, in 2 new suites : lz4 (decoding of invalid input, one-shot and streaming compression) and lz4hc (streaming, dictionaries, destSize). Tests depending on the HC level run on both levels previously selected by fuzzer. UT_sample(), KB, MB and ARRAY_SIZE() are now provided by unittest.h, for all suites. --- build/meson/meson/tests/meson.build | 2 + tests/Makefile | 2 +- tests/fuzzer.c | 702 ---------------------------- tests/unittest.c | 12 + tests/unittest.h | 12 + tests/unittest_lz4.c | 425 +++++++++++++++++ tests/unittest_lz4frame.c | 36 +- tests/unittest_lz4hc.c | 374 +++++++++++++++ 8 files changed, 837 insertions(+), 728 deletions(-) create mode 100644 tests/unittest_lz4.c create mode 100644 tests/unittest_lz4hc.c diff --git a/build/meson/meson/tests/meson.build b/build/meson/meson/tests/meson.build index 19fb035e..5884b480 100644 --- a/build/meson/meson/tests/meson.build +++ b/build/meson/meson/tests/meson.build @@ -71,6 +71,8 @@ test_exes = { 'unittest': { 'sources': files( lz4_source_root / 'tests/unittest.c', + lz4_source_root / 'tests/unittest_lz4.c', + lz4_source_root / 'tests/unittest_lz4hc.c', lz4_source_root / 'tests/unittest_lz4frame.c', lz4_source_root / 'tests/datagen.c', ), diff --git a/tests/Makefile b/tests/Makefile index 265ac22d..052acdc6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -125,7 +125,7 @@ frametest: $(eval $(call c_program_shared_o,frametest,lz4frame.o lz4file.o lz4.o lz4hc.o xxhash.o frametest.o)) unittest: -$(eval $(call c_program_shared_o,unittest,lz4frame.o lz4.o lz4hc.o xxhash.o datagen.o unittest.o unittest_lz4frame.o)) +$(eval $(call c_program_shared_o,unittest,lz4frame.o lz4.o lz4hc.o xxhash.o datagen.o unittest.o unittest_lz4.o unittest_lz4hc.o unittest_lz4frame.o)) roundTripTest: $(eval $(call c_program_shared_o,roundTripTest,lz4.o lz4hc.o xxhash.o roundTripTest.o)) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 5edb7b43..cb12fa12 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1202,159 +1202,6 @@ static void FUZ_unitTests(int compressionLevel) /* 32-bits address space overflow test */ FUZ_AddressOverflow(); - /* Test decoding with empty input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with empty input \n"); - LZ4_decompress_safe(testCompressed, testVerify, 0, testInputSize); - - /* Test decoding with a one byte input */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with one byte input \n"); - { char const tmp = (char)0xFF; - LZ4_decompress_safe(&tmp, testVerify, 1, testInputSize); - } - - /* Test decoding shortcut edge case */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with shortcut edge case \n"); - { char tmp[17]; - /* 14 bytes of literals, followed by a 14 byte match. - * Should not read beyond the end of the buffer. - * See https://github.com/lz4/lz4/issues/508. */ - *tmp = (char)0xEE; - memset(tmp + 1, 0, 14); - tmp[15] = 14; - tmp[16] = 0; - { int const r = LZ4_decompress_safe(tmp, testVerify, sizeof(tmp), testInputSize); - FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail"); - } } - - - /* Test decoding with all 0xFF input - OOB read regression test */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() with all-0xFF input \n"); - { char tmp[31]; - memset(tmp, 0xFF, sizeof(tmp)); - { int const r = LZ4_decompress_safe(tmp, testVerify, sizeof(tmp), testInputSize); - FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail on all-0xFF input"); - } } - - /* Test that ip does not go beyond iend in the fast decode loop. - * The block has 14 literals, ML_MASK match token, offset=14, one extension byte=0. - * This positions ip exactly at iend when read_variable_length reads the extension, - * then goto safe_match_copy would loop back with ip==iend, triggering assert(ip=64), - * small enough that op+length >= oend-64 triggers the safe_match_copy path. */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() fast-loop ip-at-iend regression test \n"); - { char block[18]; - char out[65]; - block[0] = (char)0xEF; /* 14 literals, ML_MASK match */ - memset(block + 1, 'A', 14); /* 14 literal bytes */ - block[15] = 0x0E; block[16] = 0x00; /* offset = 14 */ - block[17] = 0x00; /* match length extension = 0 */ - { int const r = LZ4_decompress_safe(block, out, sizeof(block), sizeof(out)); - FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail on this crafted input"); - } } - - /* Test that ip does not go beyond iend in the safe decode loop shortcut path. - * Same block as above, but output buffer is 40 bytes: small enough to skip - * the fast loop (<64), large enough for the shortcut (>=32). - * The shortcut reads literals+offset, landing ip at iend-1, then goto _copy_match - * calls read_variable_length with ip >= ilimit. */ - DISPLAYLEVEL(3, "LZ4_decompress_safe() safe-loop shortcut ip-at-iend regression test \n"); - { char block[18]; - char out[40]; - block[0] = (char)0xEF; /* 14 literals, ML_MASK match */ - memset(block + 1, 'A', 14); /* 14 literal bytes */ - block[15] = 0x0E; block[16] = 0x00; /* offset = 14 */ - block[17] = 0x00; /* match length extension = 0 */ - { int const r = LZ4_decompress_safe(block, out, sizeof(block), sizeof(out)); - FUZ_CHECKTEST(r >= 0, "LZ4_decompress_safe() should fail on this crafted input"); - } } - - /* Invalid offset 0 must not leave previous output buffer content in decoded data - * (see doc/lz4_Block_format.md). 1 literal, then a match at offset 0, - * partially decoded into a 20-byte target which ends within the match. */ - DISPLAYLEVEL(3, "LZ4_decompress_safe_partial() with offset 0 \n"); - { char block[32]; - char out[20]; - memset(block, 0, sizeof(block)); - block[0] = 0x1F; /* 1 literal, ML_MASK match */ - block[1] = 'A'; /* literal */ - block[2] = 0x00; block[3] = 0x00; /* offset = 0 (invalid) */ - block[4] = 64; /* match length extension */ - memset(out, 0xAA, sizeof(out)); - { int const r = LZ4_decompress_safe_partial(block, out, sizeof(block), sizeof(out), sizeof(out)); - int i; - for (i = 1; i < r; i++) { - FUZ_CHECKTEST(out[i] == (char)0xAA, "LZ4_decompress_safe_partial() kept previous output content at position %i, with offset 0", i); - } } } - - /* to be tested with undefined sanitizer */ - DISPLAYLEVEL(3, "LZ4_compress_default() with NULL input:"); - { int const maxCSize = LZ4_compressBound(0); - int const cSize = LZ4_compress_default(NULL, testCompressed, 0, maxCSize); - FUZ_CHECKTEST(!(cSize==1 && testCompressed[0]==0), - "compressing empty should give byte 0" - " (maxCSize == %i) (cSize == %i) (byte == 0x%02X)", - maxCSize, cSize, testCompressed[0]); - } - DISPLAYLEVEL(3, " OK \n"); - - DISPLAYLEVEL(3, "LZ4_compress_default() with both NULL input and output:"); - { int const cSize = LZ4_compress_default(NULL, NULL, 0, 0); - FUZ_CHECKTEST(cSize != 0, - "compressing into NULL must fail" - " (cSize == %i != 0)", cSize); - } - DISPLAYLEVEL(3, " OK \n"); - - DISPLAYLEVEL(3, "LZ4_compress_destSize_extState() with out-of-range acceleration : "); - { static const int accels[] = { 0, -1, -100, INT_MIN, INT_MAX }; - int const srcSize = 128 KB; - int const targetDstSize = srcSize / 4; /* < LZ4_compressBound(srcSize) : fillOutput mode */ - LZ4_stream_t state; - int refConsumed = srcSize, refSize; - U32 refCrc; - size_t n; - refSize = LZ4_compress_destSize_extState(&state, testInput, testCompressed, &refConsumed, targetDstSize, 1); - FUZ_CHECKTEST(refSize <= 0, "LZ4_compress_destSize_extState() failed"); - refCrc = XXH32(testCompressed, (size_t)refSize, 0); - for (n = 0; n < sizeof(accels)/sizeof(accels[0]); n++) { - int consumed = srcSize; - int const cSize = LZ4_compress_destSize_extState(&state, testInput, testCompressed, &consumed, targetDstSize, accels[n]); - FUZ_CHECKTEST(cSize <= 0 || cSize > targetDstSize, "acceleration %i : compression failed (%i)", accels[n], cSize); - if (accels[n] < 1) { /* replaced by the default acceleration, 1 */ - FUZ_CHECKTEST(cSize != refSize || consumed != refConsumed || XXH32(testCompressed, (size_t)cSize, 0) != refCrc, - "acceleration %i : output differs from acceleration 1", accels[n]); - } - { int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize); - FUZ_CHECKTEST(dSize != consumed || memcmp(testInput, testVerify, (size_t)consumed), - "acceleration %i : round-trip failure", accels[n]); - } } - } - DISPLAYLEVEL(3, "OK \n"); - - /* in-place compression test */ - DISPLAYLEVEL(3, "in-place compression using LZ4_compress_default() :"); - { int const sampleSize = 65 KB; - int const maxCSize = LZ4_COMPRESSBOUND(sampleSize); - int const outSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize); - int const startInputIndex = outSize - sampleSize; - char* const startInput = testCompressed + startInputIndex; - XXH32_hash_t const crcOrig = XXH32(testInput, sampleSize, 0); - int cSize; - assert(outSize < (int)testCompressedSize); - memcpy(startInput, testInput, sampleSize); /* copy at end of buffer */ - /* compress in-place */ - cSize = LZ4_compress_default(startInput, testCompressed, sampleSize, maxCSize); - assert(cSize != 0); /* ensure compression is successful */ - assert(maxCSize < INT_MAX); - assert(cSize <= maxCSize); - /* decompress and verify */ - { int const dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, testInputSize); - assert(dSize == sampleSize); /* correct size */ - { XXH32_hash_t const crcCheck = XXH32(testVerify, (size_t)dSize, 0); - FUZ_CHECKTEST(crcCheck != crcOrig, "LZ4_decompress_safe decompression corruption"); - } } } - DISPLAYLEVEL(3, " OK \n"); - /* in-place decompression test */ DISPLAYLEVEL(3, "in-place decompression, limit case:"); { int const sampleSize = 65 KB; @@ -1379,220 +1226,9 @@ static void FUZ_unitTests(int compressionLevel) } } } } } DISPLAYLEVEL(3, " OK \n"); - DISPLAYLEVEL(3, "LZ4_initStream with multiple valid alignments : "); - { typedef struct { - LZ4_stream_t state1; - LZ4_stream_t state2; - char c; - LZ4_stream_t state3; - } shct; - shct* const shc = (shct*)malloc(sizeof(*shc)); - assert(shc != NULL); - memset(shc, 0, sizeof(*shc)); - DISPLAYLEVEL(4, "state1(%p) state2(%p) state3(%p) LZ4_stream_t size(0x%x): ", - (void*)&(shc->state1), (void*)&(shc->state2), (void*)&(shc->state3), (unsigned)sizeof(LZ4_stream_t)); - FUZ_CHECKTEST( LZ4_initStream(&(shc->state1), sizeof(shc->state1)) == NULL, "state1 (%p) failed init", (void*)&(shc->state1) ); - FUZ_CHECKTEST( LZ4_initStream(&(shc->state2), sizeof(shc->state2)) == NULL, "state2 (%p) failed init", (void*)&(shc->state2) ); - FUZ_CHECKTEST( LZ4_initStream(&(shc->state3), sizeof(shc->state3)) == NULL, "state3 (%p) failed init", (void*)&(shc->state3) ); - FUZ_CHECKTEST( LZ4_initStream((char*)&(shc->state1) + 1, sizeof(shc->state1)) != NULL, - "hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->state1) + 1) ); - free(shc); - } - DISPLAYLEVEL(3, "all inits OK \n"); - - /* Allocation test */ - { LZ4_stream_t* const statePtr = LZ4_createStream(); - FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed"); - LZ4_freeStream(statePtr); - } - /* LZ4 streaming tests */ { LZ4_stream_t streamingState; - - /* simple compression test */ LZ4_initStream(&streamingState, sizeof(streamingState)); - { int const cs = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1); - FUZ_CHECKTEST(cs==0, "LZ4_compress_fast_continue() compression failed!"); - { int const r = LZ4_decompress_safe(testCompressed, testVerify, cs, testCompressedSize); - FUZ_CHECKTEST(r!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - } } - { U64 const crcOrig = XXH64(testInput, testCompressedSize, 0); - U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - } - - DISPLAYLEVEL(3, "LZ4_resetStream_fast() : a large first block gives the same output as a new stream : "); - { int const blockSize = 64 KB; /* beyond the table clearing threshold, whatever LZ4_MEMORY_USAGE */ - char* const block = (char*)malloc((size_t)blockSize); - int refSize, r; - U32 refCrc; - FUZ_CHECKTEST(block == NULL, "allocation failure"); - memcpy(block, testInput + 64 KB, (size_t)blockSize); - /* the first 4 bytes reappear, with a different 5th byte : - * a match only found when starting from a cleared table */ - memcpy(block, "ABCDx", 5); - memcpy(block + 16, "ABCDy", 5); - - LZ4_initStream(&streamingState, sizeof(streamingState)); - refSize = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1); - FUZ_CHECKTEST(refSize <= 0, "LZ4_compress_fast_continue() failed"); - refCrc = XXH32(testCompressed, (size_t)refSize, 0); - - LZ4_resetStream_fast(&streamingState); /* the state now has a history */ - r = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1); - FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc, - "output differs from a new stream (%i vs %i bytes)", r, refSize); - - LZ4_saveDict(&streamingState, testVerify, 0); /* drop history, without reset */ - r = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1); - FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc, - "after LZ4_saveDict(,,0) : output differs from a new stream (%i vs %i bytes)", r, refSize); - free(block); - } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "periodic runs : chains of short matches must not hide the period (#1805) : "); - { static const char dict[8] = { 0, 0, 0, 0, 0, 0, 0, 4 }; - char page[8 + 4096]; /* dict, then a 4 KB page alternating runs of 16 x 0xFF and 16 x 0x00 */ - LZ4_stream_t dictState; - int r; - memcpy(page, dict, sizeof(dict)); - for (r = 0; r < 32; r++) { /* several rotations compressed to ~1.5 KB, instead of < 60 bytes */ - int i, c; - for (i = 0; i < 4096; i++) - page[8 + i] = (((i + r) % 4096 + 8) / 16) % 2 == 0 ? (char)0xFF : 0; - - c = LZ4_compress_default(page, testCompressed, (int)sizeof(page), (int)testCompressedSize); - FUZ_CHECKTEST(c <= 0 || c > 100, "rotation %i, dictionary bytes in the same block : %i bytes", r, c); - - LZ4_initStream(&streamingState, sizeof(streamingState)); - LZ4_loadDict(&streamingState, dict, (int)sizeof(dict)); - c = LZ4_compress_fast_continue(&streamingState, page + 8, testCompressed, 4096, (int)testCompressedSize, 1); - FUZ_CHECKTEST(c <= 0 || c > 100, "rotation %i, LZ4_loadDict() : %i bytes", r, c); - - LZ4_initStream(&dictState, sizeof(dictState)); - LZ4_loadDict(&dictState, dict, (int)sizeof(dict)); - LZ4_initStream(&streamingState, sizeof(streamingState)); - LZ4_attach_dictionary(&streamingState, &dictState); - c = LZ4_compress_fast_continue(&streamingState, page + 8, testCompressed, 4096, (int)testCompressedSize, 1); - FUZ_CHECKTEST(c <= 0 || c > 100, "rotation %i, LZ4_attach_dictionary() : %i bytes", r, c); - } - } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "LZ4_loadDict() with a dictionary too small to be used : a large first block gives the same output as a new stream : "); - { int const blockSize = 64 KB; /* beyond the table clearing threshold, whatever LZ4_MEMORY_USAGE */ - char* const block = (char*)malloc((size_t)blockSize); - LZ4_stream_t dictStream; - int refSize, dictSize; - U32 refCrc; - FUZ_CHECKTEST(block == NULL, "allocation failure"); - memcpy(block, testInput + 64 KB, (size_t)blockSize); - /* a match only found when starting from a cleared table */ - memcpy(block, "ABCDx", 5); - memcpy(block + 16, "ABCDy", 5); - - LZ4_initStream(&streamingState, sizeof(streamingState)); - refSize = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1); - FUZ_CHECKTEST(refSize <= 0, "LZ4_compress_fast_continue() failed"); - refCrc = XXH32(testCompressed, (size_t)refSize, 0); - - /* below HASH_UNIT on all platforms : LZ4_loadDict() ignores the dictionary */ - for (dictSize = 0; dictSize < 4; dictSize++) { - int r; - LZ4_loadDict(&dictStream, testInput, dictSize); - r = LZ4_compress_fast_continue(&dictStream, block, testCompressed, blockSize, testCompressedSize, 1); - FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc, - "after LZ4_loadDict() (%i bytes) : output differs from a new stream (%i vs %i bytes)", dictSize, r, refSize); - - LZ4_loadDict(&dictStream, testInput, dictSize); - LZ4_initStream(&streamingState, sizeof(streamingState)); - LZ4_attach_dictionary(&streamingState, &dictStream); - r = LZ4_compress_fast_continue(&streamingState, block, testCompressed, blockSize, testCompressedSize, 1); - FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc, - "after LZ4_attach_dictionary() (%i bytes) : output differs from a new stream (%i vs %i bytes)", dictSize, r, refSize); - } - free(block); - } - DISPLAYLEVEL(3, "OK \n"); - - /* early saveDict */ - DISPLAYLEVEL(3, "saveDict (right after init) : "); - { LZ4_stream_t* const ctx = LZ4_initStream(&streamingState, sizeof(streamingState)); - assert(ctx != NULL); /* ensure init is successful */ - - /* Check access violation with asan */ - FUZ_CHECKTEST( LZ4_saveDict(ctx, NULL, 0) != 0, - "LZ4_saveDict() can't save anything into (NULL,0)"); - - /* Check access violation with asan */ - { char tmp_buffer[240] = { 0 }; - FUZ_CHECKTEST( LZ4_saveDict(ctx, tmp_buffer, sizeof(tmp_buffer)) != 0, - "LZ4_saveDict() can't save anything since compression hasn't started"); - } - - /* a negative size saves nothing (instead of wrapping to 64 KB), - * and the next block no longer references the previous dictionary : - * it copies the end of the dictionary, so it would match it */ - { char saveBuffer[16]; - char* const block = (char*)malloc(4 KB); - int cSize, dSize; - FUZ_CHECKTEST(block == NULL, "allocation failed"); - memcpy(block, testInput + 60 KB, 4 KB); - FUZ_CHECKTEST( LZ4_loadDict(ctx, testInput, 64 KB) <= 0, - "LZ4_loadDict() failed to prepare saveDict negative-size test"); - FUZ_CHECKTEST( LZ4_saveDict(ctx, saveBuffer, -1) != 0, - "LZ4_saveDict() must save nothing with a negative size"); - cSize = LZ4_compress_fast_continue(ctx, block, testCompressed, 4 KB, testCompressedSize, 1); - FUZ_CHECKTEST(cSize <= 0, "LZ4_compress_fast_continue() failed"); - dSize = LZ4_decompress_safe(testCompressed, testVerify, cSize, 4 KB); - FUZ_CHECKTEST(dSize != 4 KB || memcmp(testVerify, block, 4 KB), - "after LZ4_saveDict() with a negative size, the next block references the previous dictionary"); - free(block); - } - } - DISPLAYLEVEL(3, "OK \n"); - - /* the new block can overwrite the previous one (its dictionary) : any overlap must be handled */ - DISPLAYLEVEL(3, "LZ4_compress_*_continue() with input overwriting the previous block : "); - { /* block 1 : 1000 bytes at position 1000 ; block 2 : { position, size } */ - static const int overlaps[4][2] = { - { 500, 1000 }, /* covers the beginning of block 1 */ - { 1000, 1000 }, /* same place */ - { 500, 2000 }, /* covers block 1 entirely */ - { 1500, 1000 } }; /* covers the end of block 1 */ - int const pos1 = 1000, size1 = 1000; - char* const src = testVerify + 64 KB; /* content of both blocks */ - char* const cBlock2 = testCompressed + LZ4_COMPRESSBOUND(4 KB); - LZ4_streamHC_t* const sHC = LZ4_createStreamHC(); - int ov, hc, trial; - FUZ_CHECKTEST(sHC == NULL, "LZ4_createStreamHC() allocation failed"); - for (ov = 0; ov < 4; ov++) - for (hc = 0; hc < 2; hc++) - for (trial = 0; trial < 64; trial++) { - int const pos2 = overlaps[ov][0], size2 = overlaps[ov][1]; - LZ4_streamDecode_t sd; - int c1, c2, i, r; - for (i = 0; i < size1 + size2; i++) src[i] = "ACGT"[FUZ_rand(&randState) & 3]; /* many matches */ - LZ4_initStream(&streamingState, sizeof(streamingState)); - LZ4_initStreamHC(sHC, sizeof(*sHC)); - memcpy(ringBuffer + pos1, src, (size_t)size1); - c1 = hc ? LZ4_compress_HC_continue(sHC, ringBuffer + pos1, testCompressed, size1, LZ4_COMPRESSBOUND(4 KB)) - : LZ4_compress_fast_continue(&streamingState, ringBuffer + pos1, testCompressed, size1, LZ4_COMPRESSBOUND(4 KB), 1); - memcpy(ringBuffer + pos2, src + size1, (size_t)size2); - c2 = hc ? LZ4_compress_HC_continue(sHC, ringBuffer + pos2, cBlock2, size2, LZ4_COMPRESSBOUND(4 KB)) - : LZ4_compress_fast_continue(&streamingState, ringBuffer + pos2, cBlock2, size2, LZ4_COMPRESSBOUND(4 KB), 1); - FUZ_CHECKTEST(c1 <= 0 || c2 <= 0, "compression failed"); - LZ4_setStreamDecode(&sd, NULL, 0); - r = LZ4_decompress_safe_continue(&sd, testCompressed, testVerify, c1, size1); - r += LZ4_decompress_safe_continue(&sd, cBlock2, testVerify + size1, c2, size2); - FUZ_CHECKTEST(r != size1 + size2 || memcmp(testVerify, src, (size_t)(size1 + size2)), - "%s : corrupted output, block 2 (position %i, size %i) overwrites block 1 (position %i, size %i)", - hc ? "LZ4_compress_HC_continue()" : "LZ4_compress_fast_continue()", pos2, size2, pos1, size1); - } - LZ4_freeStreamHC(sHC); - } - DISPLAYLEVEL(3, "OK \n"); /* ring buffer test */ { XXH64_state_t xxhOrig; @@ -1646,270 +1282,11 @@ static void FUZ_unitTests(int compressionLevel) } } } - DISPLAYLEVEL(3, "LZ4_compress_HC() with NULL input:"); - { int const maxCSize = LZ4_compressBound(0); - int level; - for (level=LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++) { - int const cSize = LZ4_compress_HC(NULL, testCompressed, 0, maxCSize, level); - FUZ_CHECKTEST(!(cSize==1 && testCompressed[0]==0), - "compressing empty should give byte 0" - " (maxCSize == %i) (cSize == %i) (byte == 0x%02X)", - maxCSize, cSize, testCompressed[0]); - } - } - DISPLAYLEVEL(3, " OK \n"); - - DISPLAYLEVEL(3, "LZ4_compress_HC() with both NULL input and output:"); - { int level; - for (level=LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++) { - int const cSize = LZ4_compress_HC(NULL, NULL, 0, 0, level); - FUZ_CHECKTEST(cSize != 0, - "compressing into NULL must fail" - " (cSize == %i != 0)", cSize); - } - } - DISPLAYLEVEL(3, " OK \n"); - - DISPLAYLEVEL(3, "LZ4_initStreamHC with multiple valid alignments : "); - { typedef struct { - LZ4_streamHC_t hc1; - LZ4_streamHC_t hc2; - char c; - LZ4_streamHC_t hc3; - } shct; - shct* const shc = (shct*)malloc(sizeof(*shc)); - assert(shc != NULL); - memset(shc, 0, sizeof(*shc)); - DISPLAYLEVEL(4, "hc1(%p) hc2(%p) hc3(%p) size(0x%x): ", - (void*)&(shc->hc1), (void*)&(shc->hc2), (void*)&(shc->hc3), - (unsigned)sizeof(LZ4_streamHC_t)); - FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc1), sizeof(shc->hc1)) == NULL, "hc1 (%p) failed init", (void*)&(shc->hc1) ); - FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc2), sizeof(shc->hc2)) == NULL, "hc2 (%p) failed init", (void*)&(shc->hc2) ); - FUZ_CHECKTEST( LZ4_initStreamHC(&(shc->hc3), sizeof(shc->hc3)) == NULL, "hc3 (%p) failed init", (void*)&(shc->hc3) ); - FUZ_CHECKTEST( LZ4_initStreamHC((char*)&(shc->hc1) + 1, sizeof(shc->hc1)) != NULL, - "hc1+1 (%p) init must fail, due to bad alignment", (void*)((char*)&(shc->hc1) + 1) ); - free(shc); - } - DISPLAYLEVEL(3, "all inits OK \n"); - /* LZ4 HC streaming tests */ { LZ4_streamHC_t sHC; /* statically allocated */ int result; LZ4_initStreamHC(&sHC, sizeof(sHC)); - /* Allocation test */ - DISPLAYLEVEL(3, "Basic HC allocation : "); - { LZ4_streamHC_t* const sp = LZ4_createStreamHC(); - FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); - LZ4_freeStreamHC(sp); - } - DISPLAYLEVEL(3, "OK \n"); - - /* simple HC compression test */ - DISPLAYLEVEL(3, "Simple HC round-trip : "); - { U64 const crc64 = XXH64(testInput, testCompressedSize, 0); - LZ4_setCompressionLevel(&sHC, compressionLevel); - result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() decompression corruption"); - } } - DISPLAYLEVEL(3, "OK \n"); - - /* deprecated streaming API */ - DISPLAYLEVEL(3, "LZ4_compressHC2_continue() round-trip : "); - { int const srcSize = 64 KB; /* LZ4_compressBound(srcSize) fits into testCompressed */ - void* const ctx = LZ4_createHC(testInput); - FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed"); - result = LZ4_compressHC2_continue(ctx, testInput, testCompressed, srcSize, compressionLevel); - FUZ_CHECKTEST(result<=0, "LZ4_compressHC2_continue() compression failed"); - result = LZ4_decompress_safe(testCompressed, testVerify, result, srcSize); - FUZ_CHECKTEST(result!=srcSize || memcmp(testInput, testVerify, (size_t)srcSize), - "LZ4_compressHC2_continue() : corrupted round-trip"); - LZ4_freeHC(ctx); - } - DISPLAYLEVEL(3, "OK \n"); - - DISPLAYLEVEL(3, "level 2 : the match tested at ip+1 must not be ip itself, inside runs : "); - { char runs[2 + 4096]; /* 0xFF 0x00, then runs of 10 x 0xFF and 10 x 0x00 */ - int i; - runs[0] = (char)0xFF; runs[1] = 0; - for (i = 0; i < 4096; i++) runs[2 + i] = (i % 20) < 10 ? (char)0xFF : 0; - result = LZ4_compress_HC(runs, testCompressed, (int)sizeof(runs), (int)testCompressedSize, 2); - /* ~1.6 KB (one literal and one offset 1 per run), instead of < 50 bytes */ - FUZ_CHECKTEST(result <= 0 || result > 100, "%i bytes", result); - } - DISPLAYLEVEL(3, "OK \n"); - - /* saveDictHC test #926 */ - DISPLAYLEVEL(3, "saveDictHC test #926 : "); - { LZ4_streamHC_t* const ctx = LZ4_initStreamHC(&sHC, sizeof(sHC)); - assert(ctx != NULL); /* ensure init is successful */ - - /* Check access violation with asan */ - FUZ_CHECKTEST( LZ4_saveDictHC(ctx, NULL, 0) != 0, - "LZ4_saveDictHC() can't save anything into (NULL,0)"); - - /* Check access violation with asan */ - { char tmp_buffer[240] = { 0 }; - FUZ_CHECKTEST( LZ4_saveDictHC(ctx, tmp_buffer, sizeof(tmp_buffer)) != 0, - "LZ4_saveDictHC() can't save anything since compression hasn't started"); - } - - /* next block is compressed as the first one of the stream, - * without (attach==0) or with (attach==1) an attached dictionary */ - { LZ4_streamHC_t* const dictHC = LZ4_createStreamHC(); - int const levels[] = { LZ4HC_CLEVEL_DEFAULT, 3, LZ4HC_CLEVEL_OPT_MIN, LZ4HC_CLEVEL_MAX }; /* first : level of LZ4_initStreamHC() */ - int attach, l; - FUZ_CHECKTEST(dictHC == NULL, "LZ4_createStreamHC() allocation failed"); - for (l = 0; l < (int)(sizeof(levels) / sizeof(levels[0])); l++) { - for (attach = 0; attach <= 1; attach++) { - char saveBuffer[240]; - int cSize, dSize; - if (l || attach) LZ4_resetStreamHC_fast(ctx, levels[l]); /* first test : right after init */ - if (attach) { - LZ4_resetStreamHC_fast(dictHC, levels[l]); - FUZ_CHECKTEST(LZ4_loadDictHC(dictHC, testInput, 64 KB) != 64 KB, "LZ4_loadDictHC() failed"); - LZ4_attach_HC_dictionary(ctx, dictHC); - } - FUZ_CHECKTEST( LZ4_saveDictHC(ctx, saveBuffer, sizeof(saveBuffer)) != 0, - "LZ4_saveDictHC() can't save anything since compression hasn't started"); - cSize = LZ4_compress_HC_continue(ctx, testInput + 64 KB, testCompressed, 64 KB, testCompressedSize); - FUZ_CHECKTEST(cSize <= 0, "LZ4_compress_HC_continue() failed"); - dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, 64 KB, - testInput, attach ? 64 KB : 0); - FUZ_CHECKTEST(dSize != 64 KB || memcmp(testVerify, testInput + 64 KB, 64 KB), - "level %i, dictionary %i : block compressed after LZ4_saveDictHC() is corrupted", levels[l], attach); - } } - LZ4_freeStreamHC(dictHC); - } } - DISPLAYLEVEL(3, "OK \n"); - - /* long sequence test */ - DISPLAYLEVEL(3, "Long sequence HC_destSize test : "); - { size_t const blockSize = 1 MB; - size_t const targetSize = 4116; /* size carefully selected to trigger an overflow */ - void* const block = malloc(blockSize); - void* const dstBlock = malloc(targetSize+1); - BYTE const sentinel = 101; - int srcSize; - - assert(block != NULL); assert(dstBlock != NULL); - memset(block, 0, blockSize); - ((char*)dstBlock)[targetSize] = sentinel; - - LZ4_resetStreamHC_fast(&sHC, 3); - assert(blockSize < INT_MAX); - srcSize = (int)blockSize; - assert(targetSize < INT_MAX); - result = LZ4_compress_HC_destSize(&sHC, (const char*)block, (char*)dstBlock, &srcSize, (int)targetSize, 3); - DISPLAYLEVEL(4, "cSize=%i; readSize=%i; ", result, srcSize); - FUZ_CHECKTEST(result != 4116, "LZ4_compress_HC_destSize() : " - "compression (%i->%i) must fill dstBuffer (%i) exactly", - srcSize, result, (int)targetSize); - FUZ_CHECKTEST(((char*)dstBlock)[targetSize] != sentinel, - "LZ4_compress_HC_destSize() overwrites dst buffer"); - FUZ_CHECKTEST(srcSize < 1045000, "LZ4_compress_HC_destSize() doesn't compress enough" - " (%i -> %i , expected > %i)", srcSize, result, 1045000); - - LZ4_resetStreamHC_fast(&sHC, 3); /* make sure the context is clean after the test */ - free(block); - free(dstBlock); - } - DISPLAYLEVEL(3, " OK \n"); - - /* a failed compression marks the state as dirty, which is fully re-initialized at next reset */ - DISPLAYLEVEL(3, "favorDecSpeed survives a failed compression : "); - { int const srcSize = 64 KB; - int const level = LZ4HC_CLEVEL_OPT_MIN; - int refSize, r; - U32 refCrc; - - /* reference : favor decompression speed */ - LZ4_initStreamHC(&sHC, sizeof(sHC)); - LZ4_resetStreamHC_fast(&sHC, level); - LZ4_favorDecompressionSpeed(&sHC, 1); - refSize = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, srcSize, testCompressedSize); - FUZ_CHECKTEST(refSize <= 0, "LZ4_compress_HC_continue() failed"); - refCrc = XXH32(testCompressed, (size_t)refSize, 0); - - /* this test is only meaningful if favorDecSpeed changes the output */ - LZ4_resetStreamHC_fast(&sHC, level); - LZ4_favorDecompressionSpeed(&sHC, 0); - r = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, srcSize, testCompressedSize); - FUZ_CHECKTEST(r == refSize && XXH32(testCompressed, (size_t)r, 0) == refCrc, - "favorDecSpeed doesn't change the output of this test input"); - - /* failed compression */ - LZ4_resetStreamHC_fast(&sHC, level); - LZ4_favorDecompressionSpeed(&sHC, 1); - r = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, srcSize, refSize - 1); - FUZ_CHECKTEST(r != 0, "compression into a too small buffer should fail"); - - /* next stream : must still favor decompression speed */ - LZ4_resetStreamHC_fast(&sHC, level); - r = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, srcSize, testCompressedSize); - FUZ_CHECKTEST(r != refSize || XXH32(testCompressed, (size_t)r, 0) != refCrc, - "favorDecSpeed lost after a failed compression (%i vs %i bytes)", r, refSize); - } - DISPLAYLEVEL(3, "OK \n"); - - /* simple dictionary HC compression test */ - DISPLAYLEVEL(3, "HC dictionary compression test : "); - { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 64 KB); - { int const cSize = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(cSize==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : @return = %i", cSize); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - { int const dSize = LZ4_decompress_safe_usingDict(testCompressed, testVerify, cSize, testCompressedSize, testInput, 64 KB); - FUZ_CHECKTEST(dSize!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed"); - } } - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); - } } - DISPLAYLEVEL(3, " OK \n"); - - /* multiple HC compression test with dictionary */ - { int result1, result2; - int segSize = testCompressedSize / 2; - XXH64_hash_t const crc64 = ( (void)assert((unsigned)segSize + testCompressedSize < testInputSize) , - XXH64(testInput + segSize, testCompressedSize, 0) ); - LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, segSize); - result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1); - FUZ_CHECKTEST(result1==0, "LZ4_compress_HC_continue() dictionary compression failed : result = %i", result1); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*(size_t)segSize, testCompressed+result1, segSize, segSize-1); - FUZ_CHECKTEST(result2==0, "LZ4_compress_HC_continue() dictionary compression failed : result = %i", result2); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed"); - result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize); - FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed"); - { XXH64_hash_t const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); - } } - - /* remote dictionary HC compression test */ - { U64 const crc64 = XXH64(testInput + 64 KB, testCompressedSize, 0); - LZ4_resetStreamHC_fast(&sHC, compressionLevel); - LZ4_loadDictHC(&sHC, testInput, 32 KB); - result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1); - FUZ_CHECKTEST(result==0, "LZ4_compress_HC_continue() remote dictionary failed : result = %i", result); - FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean"); - - result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB); - FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test"); - { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0); - FUZ_CHECKTEST(crc64!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); - } } - /* multiple HC compression with ext. dictionary */ { XXH64_state_t crcOrigState; XXH64_state_t crcNewState; @@ -2222,85 +1599,6 @@ static void FUZ_unitTests(int compressionLevel) DISPLAYLEVEL(3, "OK \n"); } - DISPLAYLEVEL(3, "LZ4_compress_HC_destSize : "); - /* encode congenerical sequence test for HC compressors */ - { LZ4_streamHC_t* const sHC = LZ4_createStreamHC(); - int const src_buf_size = 3 MB; - int const dst_buf_size = 6 KB; - int const payload = 0; - int const dst_step = 43; - int const dst_min_len = 33 + (FUZ_rand(&randState) % dst_step); - int const dst_max_len = 5000; - int slen, dlen; - char* sbuf1 = (char*)malloc(src_buf_size + 1); - char* sbuf2 = (char*)malloc(src_buf_size + 1); - char* dbuf1 = (char*)malloc(dst_buf_size + 1); - char* dbuf2 = (char*)malloc(dst_buf_size + 1); - - assert(sHC != NULL); - assert(dst_buf_size > dst_max_len); - if (!sbuf1 || !sbuf2 || !dbuf1 || !dbuf2) { - EXIT_MSG("not enough memory for FUZ_unitTests (destSize)"); - } - for (dlen = dst_min_len; dlen <= dst_max_len; dlen += dst_step) { - int src_len = (dlen - 10)*255 + 24; - if (src_len + 10 >= src_buf_size) break; /* END of check */ - for (slen = src_len - 3; slen <= src_len + 3; slen++) { - int srcsz1, srcsz2; - int dsz1, dsz2; - int res1, res2; - char const endchk = (char)0x88; - DISPLAYLEVEL(5, "slen = %i, ", slen); - - srcsz1 = slen; - memset(sbuf1, payload, slen); - memset(dbuf1, 0, dlen); - dbuf1[dlen] = endchk; - dsz1 = LZ4_compress_destSize(sbuf1, dbuf1, &srcsz1, dlen); - DISPLAYLEVEL(5, "LZ4_compress_destSize: %i bytes compressed into %i bytes, ", srcsz1, dsz1); - DISPLAYLEVEL(5, "last token : 0x%0X, ", dbuf1[dsz1 - 6]); - DISPLAYLEVEL(5, "last ML extra lenbyte : 0x%0X, \n", dbuf1[dsz1 - 7]); - FUZ_CHECKTEST(dbuf1[dlen] != endchk, "LZ4_compress_destSize() overwrite dst buffer !"); - FUZ_CHECKTEST(dsz1 <= 0, "LZ4_compress_destSize() compression failed"); - FUZ_CHECKTEST(dsz1 > dlen, "LZ4_compress_destSize() result larger than dst buffer !"); - FUZ_CHECKTEST(srcsz1 > slen, "LZ4_compress_destSize() read more than src buffer !"); - - res1 = LZ4_decompress_safe(dbuf1, sbuf1, dsz1, src_buf_size); - FUZ_CHECKTEST(res1 != srcsz1, "LZ4_compress_destSize() decompression failed!"); - - srcsz2 = slen; - memset(sbuf2, payload, slen); - memset(dbuf2, 0, dlen); - dbuf2[dlen] = endchk; - LZ4_resetStreamHC(sHC, compressionLevel); - dsz2 = LZ4_compress_HC_destSize(sHC, sbuf2, dbuf2, &srcsz2, dlen, compressionLevel); - DISPLAYLEVEL(5, "LZ4_compress_HC_destSize: %i bytes compressed into %i bytes, ", srcsz2, dsz2); - DISPLAYLEVEL(5, "last token : 0x%0X, ", dbuf2[dsz2 - 6]); - DISPLAYLEVEL(5, "last ML extra lenbyte : 0x%0X, \n", dbuf2[dsz2 - 7]); - FUZ_CHECKTEST(dbuf2[dlen] != endchk, "LZ4_compress_HC_destSize() overwrite dst buffer !"); - FUZ_CHECKTEST(dsz2 <= 0, "LZ4_compress_HC_destSize() compression failed"); - FUZ_CHECKTEST(dsz2 > dlen, "LZ4_compress_HC_destSize() result larger than dst buffer !"); - FUZ_CHECKTEST(srcsz2 > slen, "LZ4_compress_HC_destSize() read more than src buffer !"); - FUZ_CHECKTEST(dsz2 != dsz1, "LZ4_compress_HC_destSize() return incorrect result !"); - FUZ_CHECKTEST(srcsz2 != srcsz1, "LZ4_compress_HC_destSize() return incorrect src buffer size " - ": srcsz2(%i) != srcsz1(%i)", srcsz2, srcsz1); - FUZ_CHECKTEST(memcmp(dbuf2, dbuf1, (size_t)dsz2), "LZ4_compress_HC_destSize() return incorrect data into dst buffer !"); - - res2 = LZ4_decompress_safe(dbuf2, sbuf1, dsz2, src_buf_size); - FUZ_CHECKTEST(res2 != srcsz1, "LZ4_compress_HC_destSize() decompression failed!"); - - FUZ_CHECKTEST(memcmp(sbuf1, sbuf2, (size_t)res2), "LZ4_compress_HC_destSize() decompression corruption!"); - } - } - LZ4_freeStreamHC(sHC); - free(sbuf1); - free(sbuf2); - free(dbuf1); - free(dbuf2); - } - DISPLAYLEVEL(3, " OK \n"); - - /* clean up */ free(testInput); free(testCompressed); diff --git a/tests/unittest.c b/tests/unittest.c index 12600837..77c62256 100644 --- a/tests/unittest.c +++ b/tests/unittest.c @@ -24,6 +24,7 @@ */ #include "unittest.h" +#include "datagen.h" /* RDG_genBuffer */ #include /* setjmp, longjmp */ #include /* va_list */ #include /* printf, snprintf */ @@ -35,9 +36,13 @@ /*-************************************ * Test suites : one per test file **************************************/ +extern const UT_suite UT_suite_lz4; +extern const UT_suite UT_suite_lz4hc; extern const UT_suite UT_suite_lz4frame; static const UT_suite* const g_suites[] = { + &UT_suite_lz4, + &UT_suite_lz4hc, &UT_suite_lz4frame, }; @@ -80,6 +85,13 @@ unsigned UT_rand(void) return g_randState; } +void* UT_sample(size_t size) +{ + void* const buffer = UT_malloc(size); + RDG_genBuffer(buffer, size, 0.5, 0.0, UT_rand()); + return buffer; +} + /*-************************************ * Assertions diff --git a/tests/unittest.h b/tests/unittest.h index 5b1163d5..9c86e695 100644 --- a/tests/unittest.h +++ b/tests/unittest.h @@ -66,6 +66,14 @@ #endif +/*-************************************ +* Common definitions +**************************************/ +#define KB *(1U<<10) +#define MB *(1U<<20) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) + + /*-************************************ * Tests **************************************/ @@ -111,6 +119,10 @@ void* UT_defer(void (*release)(void*), void* ptr); * pseudo-random generator, restarted from the same seed (-s#) for each test */ unsigned UT_rand(void); +/* UT_sample() : + * @return : @size bytes of compressible data, generated from UT_rand(), released at the end of the test */ +void* UT_sample(size_t size); + /*-************************************ * Implementation of assertions diff --git a/tests/unittest_lz4.c b/tests/unittest_lz4.c new file mode 100644 index 00000000..97302f0d --- /dev/null +++ b/tests/unittest_lz4.c @@ -0,0 +1,425 @@ +/* + unittest_lz4 - unit tests of lz4 (block format, fast levels) + Copyright (C) Yann Collet 2026 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +#include "unittest.h" +#include /* INT_MIN, INT_MAX */ +#include /* memset, memcpy */ +#define LZ4_STATIC_LINKING_ONLY /* LZ4_compress_destSize_extState, LZ4_attach_dictionary */ +#include "lz4.h" +#include "lz4hc.h" + + +/*-************************************ +* Helpers +**************************************/ +static void freeStream(void* stream) { LZ4_freeStream((LZ4_stream_t*)stream); } +static void freeStreamHC(void* stream) { LZ4_freeStreamHC((LZ4_streamHC_t*)stream); } + +/* initialized stream, released at the end of the test */ +static LZ4_stream_t* newStream(void) +{ + LZ4_stream_t* const stream = (LZ4_stream_t*)UT_defer(freeStream, LZ4_createStream()); + ASSERT(stream != NULL); + return stream; +} + +/* 64 KB block, beyond the table clearing threshold whatever LZ4_MEMORY_USAGE, + * in which the first 4 bytes reappear with a different 5th byte : + * a match only found when starting from a cleared table */ +#define BLOCK_64K (64 KB) +static char* clearedTableBlock(void) +{ + char* const block = (char*)UT_sample(BLOCK_64K); + memcpy(block, "ABCDx", 5); + memcpy(block + 16, "ABCDy", 5); + return block; +} + +/* large enough to enter any decoding loop */ +#define DECODED_CAPACITY (64 KB) + + +/*-************************************ +* Decoding of invalid input +**************************************/ +TEST(decompress_safe_emptyInput) +{ + char const src[1] = { 0 }; + char* const decoded = (char*)UT_malloc(DECODED_CAPACITY); + ASSERT(LZ4_decompress_safe(src, decoded, 0, DECODED_CAPACITY) < 0); +} + +TEST(decompress_safe_oneByteInput) +{ + char const src[1] = { (char)0xFF }; + char* const decoded = (char*)UT_malloc(DECODED_CAPACITY); + ASSERT(LZ4_decompress_safe(src, decoded, 1, DECODED_CAPACITY) < 0); +} + +/* 14 bytes of literals, followed by a 14 byte match : + * must not read beyond the end of the input (#508) */ +TEST(decompress_safe_shortcutEdgeCase) +{ + char block[17]; + char* const decoded = (char*)UT_malloc(DECODED_CAPACITY); + block[0] = (char)0xEE; + memset(block + 1, 0, 14); + block[15] = 14; + block[16] = 0; + ASSERT(LZ4_decompress_safe(block, decoded, sizeof(block), DECODED_CAPACITY) < 0); +} + +/* out-of-bounds read regression test */ +TEST(decompress_safe_all0xFF) +{ + char block[31]; + char* const decoded = (char*)UT_malloc(DECODED_CAPACITY); + memset(block, 0xFF, sizeof(block)); + ASSERT(LZ4_decompress_safe(block, decoded, sizeof(block), DECODED_CAPACITY) < 0); +} + +/* ip must not go beyond iend. + * The block has 14 literals, ML_MASK match token, offset=14, one extension byte=0, + * which positions ip exactly at iend when read_variable_length() reads the extension. + * - 65 bytes output : large enough to enter the fast loop (>=64), + * small enough that op+length >= oend-64 triggers the safe_match_copy path, + * which would loop back with ip==iend, triggering assert(ip=32), + * which reads literals+offset, landing ip at iend-1, then goto _copy_match + * calls read_variable_length() with ip >= ilimit. */ +TEST(decompress_safe_ipAtIend) +{ + static const int dstCapacities[] = { 65, 40 }; + char block[18]; + char decoded[65]; + size_t n; + block[0] = (char)0xEF; /* 14 literals, ML_MASK match */ + memset(block + 1, 'A', 14); /* 14 literal bytes */ + block[15] = 0x0E; block[16] = 0x00; /* offset = 14 */ + block[17] = 0x00; /* match length extension = 0 */ + for (n = 0; n < ARRAY_SIZE(dstCapacities); n++) + ASSERT(LZ4_decompress_safe(block, decoded, sizeof(block), dstCapacities[n]) < 0); +} + +/* Invalid offset 0 must not leave previous output buffer content in decoded data + * (see doc/lz4_Block_format.md). 1 literal, then a match at offset 0, + * partially decoded into a 20-byte target which ends within the match. */ +TEST(decompress_safe_partial_offset0) +{ + char block[32]; + char decoded[20]; + int r, i; + memset(block, 0, sizeof(block)); + block[0] = 0x1F; /* 1 literal, ML_MASK match */ + block[1] = 'A'; /* literal */ + block[2] = 0x00; block[3] = 0x00; /* offset = 0 (invalid) */ + block[4] = 64; /* match length extension */ + memset(decoded, 0xAA, sizeof(decoded)); + r = LZ4_decompress_safe_partial(block, decoded, sizeof(block), sizeof(decoded), sizeof(decoded)); + for (i = 1; i < r; i++) + ASSERT_MSG(decoded[i] != (char)0xAA, "previous content kept at position %i", i); +} + + +/*-************************************ +* One-shot compression +**************************************/ +/* to be tested with UBSan */ +TEST(compress_default_nullInput) +{ + char dst[16]; + int const cSize = LZ4_compress_default(NULL, dst, 0, LZ4_compressBound(0)); + ASSERT_EQ(cSize, 1); + ASSERT_EQ(dst[0], 0); +} + +TEST(compress_default_nullInputAndOutput) +{ + ASSERT_EQ(LZ4_compress_default(NULL, NULL, 0, 0), 0); +} + +/* acceleration < 1 is replaced by the default acceleration, 1 */ +TEST(compress_destSize_extState_outOfRangeAcceleration) +{ + static const int accels[] = { 0, -1, -100, INT_MIN, INT_MAX }; + int const srcSize = 128 KB; + int const targetDstSize = srcSize / 4; /* < LZ4_compressBound(srcSize) : fillOutput mode */ + const char* const src = (const char*)UT_sample((size_t)srcSize); + char* const ref = (char*)UT_malloc((size_t)targetDstSize); + char* const dst = (char*)UT_malloc((size_t)targetDstSize); + char* const decoded = (char*)UT_malloc((size_t)srcSize); + LZ4_stream_t state; + int refConsumed = srcSize, refSize; + size_t n; + refSize = LZ4_compress_destSize_extState(&state, src, ref, &refConsumed, targetDstSize, 1); + ASSERT(refSize > 0); + for (n = 0; n < ARRAY_SIZE(accels); n++) { + int consumed = srcSize; + int const cSize = LZ4_compress_destSize_extState(&state, src, dst, &consumed, targetDstSize, accels[n]); + ASSERT_MSG(cSize > 0 && cSize <= targetDstSize, "acceleration %i : compression failed (%i)", accels[n], cSize); + if (accels[n] < 1) { + ASSERT_MSG(cSize == refSize && consumed == refConsumed && !memcmp(dst, ref, (size_t)cSize), + "acceleration %i : output differs from acceleration 1", accels[n]); + } + ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, srcSize), consumed); + ASSERT_MEMEQ(decoded, src, (size_t)consumed); + } +} + +TEST(compress_default_inPlace) +{ + int const srcSize = 65 KB; + int const maxCSize = LZ4_COMPRESSBOUND(srcSize); + int const bufferSize = LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCSize); + const char* const src = (const char*)UT_sample((size_t)srcSize); + char* const buffer = (char*)UT_malloc((size_t)bufferSize); + char* const input = buffer + bufferSize - srcSize; /* at the end of the buffer */ + char* const decoded = (char*)UT_malloc((size_t)srcSize); + int cSize; + memcpy(input, src, (size_t)srcSize); + cSize = LZ4_compress_default(input, buffer, srcSize, maxCSize); + ASSERT(cSize > 0); + ASSERT_EQ(LZ4_decompress_safe(buffer, decoded, cSize, srcSize), srcSize); + ASSERT_MEMEQ(decoded, src, (size_t)srcSize); +} + + +/*-************************************ +* Streaming compression +**************************************/ +TEST(initStream_alignment) +{ + typedef struct { + LZ4_stream_t state1; + LZ4_stream_t state2; + char c; + LZ4_stream_t state3; + } shct; + shct* const shc = (shct*)UT_malloc(sizeof(shct)); + memset(shc, 0, sizeof(*shc)); + ASSERT(LZ4_initStream(&shc->state1, sizeof(shc->state1)) != NULL); + ASSERT(LZ4_initStream(&shc->state2, sizeof(shc->state2)) != NULL); + ASSERT(LZ4_initStream(&shc->state3, sizeof(shc->state3)) != NULL); + ASSERT(LZ4_initStream((char*)&shc->state1 + 1, sizeof(shc->state1)) == NULL); /* bad alignment */ +} + +TEST(createStream) +{ + newStream(); +} + +TEST(compress_fast_continue_roundTrip) +{ + int const srcSize = 130 KB; + const char* const src = (const char*)UT_sample((size_t)srcSize); + char* const dst = (char*)UT_malloc((size_t)srcSize); + char* const decoded = (char*)UT_malloc((size_t)srcSize); + int const cSize = LZ4_compress_fast_continue(newStream(), src, dst, srcSize, srcSize - 1, 1); + ASSERT(cSize > 0); + ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, srcSize), srcSize); + ASSERT_MEMEQ(decoded, src, (size_t)srcSize); +} + +TEST(resetStream_fast_largeFirstBlock) /* a large first block gives the same output as a new stream */ +{ + const char* const block = clearedTableBlock(); + int const capacity = LZ4_COMPRESSBOUND(BLOCK_64K); + char* const ref = (char*)UT_malloc((size_t)capacity); + char* const dst = (char*)UT_malloc((size_t)capacity); + char saveBuffer[1]; + LZ4_stream_t* const stream = newStream(); + int const refSize = LZ4_compress_fast_continue(stream, block, ref, BLOCK_64K, capacity, 1); + ASSERT(refSize > 0); + + LZ4_resetStream_fast(stream); /* the state now has a history */ + ASSERT_EQ(LZ4_compress_fast_continue(stream, block, dst, BLOCK_64K, capacity, 1), refSize); + ASSERT_MEMEQ(dst, ref, (size_t)refSize); + + LZ4_saveDict(stream, saveBuffer, 0); /* drop history, without reset */ + ASSERT_EQ(LZ4_compress_fast_continue(stream, block, dst, BLOCK_64K, capacity, 1), refSize); + ASSERT_MEMEQ(dst, ref, (size_t)refSize); +} + +/* a dictionary below HASH_UNIT on all platforms is ignored by LZ4_loadDict() : + * a large first block gives the same output as a new stream */ +TEST(loadDict_tooSmall) +{ + const char* const block = clearedTableBlock(); + const char* const dict = (const char*)UT_sample(4); + int const capacity = LZ4_COMPRESSBOUND(BLOCK_64K); + char* const ref = (char*)UT_malloc((size_t)capacity); + char* const dst = (char*)UT_malloc((size_t)capacity); + LZ4_stream_t* const stream = newStream(); + LZ4_stream_t* const dictStream = newStream(); + int const refSize = LZ4_compress_fast_continue(stream, block, ref, BLOCK_64K, capacity, 1); + int dictSize; + ASSERT(refSize > 0); + for (dictSize = 0; dictSize < 4; dictSize++) { + LZ4_loadDict(dictStream, dict, dictSize); + ASSERT_MSG(LZ4_compress_fast_continue(dictStream, block, dst, BLOCK_64K, capacity, 1) == refSize + && !memcmp(dst, ref, (size_t)refSize), + "after LZ4_loadDict() (%i bytes) : output differs from a new stream", dictSize); + + LZ4_loadDict(dictStream, dict, dictSize); + LZ4_initStream(stream, sizeof(*stream)); + LZ4_attach_dictionary(stream, dictStream); + ASSERT_MSG(LZ4_compress_fast_continue(stream, block, dst, BLOCK_64K, capacity, 1) == refSize + && !memcmp(dst, ref, (size_t)refSize), + "after LZ4_attach_dictionary() (%i bytes) : output differs from a new stream", dictSize); + } +} + +TEST(saveDict_afterInit) +{ + LZ4_stream_t* const stream = newStream(); + char saveBuffer[240]; + ASSERT_EQ(LZ4_saveDict(stream, NULL, 0), 0); + ASSERT_EQ(LZ4_saveDict(stream, saveBuffer, sizeof(saveBuffer)), 0); /* compression hasn't started */ +} + +/* a negative size saves nothing (instead of wrapping to 64 KB), + * and the next block no longer references the previous dictionary : + * it copies the end of the dictionary, so it would match it */ +TEST(saveDict_negativeSize) +{ + const char* const dict = (const char*)UT_sample(64 KB); + char* const block = (char*)UT_malloc(4 KB); + int const capacity = LZ4_COMPRESSBOUND(4 KB); + char* const dst = (char*)UT_malloc((size_t)capacity); + char* const decoded = (char*)UT_malloc(4 KB); + LZ4_stream_t* const stream = newStream(); + char saveBuffer[16]; + int cSize; + memcpy(block, dict + 60 KB, 4 KB); + ASSERT(LZ4_loadDict(stream, dict, 64 KB) > 0); + ASSERT_EQ(LZ4_saveDict(stream, saveBuffer, -1), 0); + cSize = LZ4_compress_fast_continue(stream, block, dst, 4 KB, capacity, 1); + ASSERT(cSize > 0); + ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, 4 KB), 4 KB); /* no reference to the dictionary */ + ASSERT_MEMEQ(decoded, block, 4 KB); +} + +/* chains of short matches must not hide the period (#1805) */ +TEST(compress_periodicRuns) +{ + static const char dict[8] = { 0, 0, 0, 0, 0, 0, 0, 4 }; + char page[8 + 4096]; /* dict, then a 4 KB page alternating runs of 16 x 0xFF and 16 x 0x00 */ + int const capacity = LZ4_COMPRESSBOUND(sizeof(page)); + char* const dst = (char*)UT_malloc((size_t)capacity); + LZ4_stream_t* const stream = newStream(); + LZ4_stream_t* const dictStream = newStream(); + int r; + memcpy(page, dict, sizeof(dict)); + for (r = 0; r < 32; r++) { /* several rotations compressed to ~1.5 KB, instead of < 60 bytes */ + int i, c; + for (i = 0; i < 4096; i++) + page[8 + i] = (((i + r) % 4096 + 8) / 16) % 2 == 0 ? (char)0xFF : 0; + + c = LZ4_compress_default(page, dst, (int)sizeof(page), capacity); + ASSERT_MSG(c > 0 && c <= 100, "rotation %i, dictionary bytes in the same block : %i bytes", r, c); + + LZ4_initStream(stream, sizeof(*stream)); + LZ4_loadDict(stream, dict, (int)sizeof(dict)); + c = LZ4_compress_fast_continue(stream, page + 8, dst, 4096, capacity, 1); + ASSERT_MSG(c > 0 && c <= 100, "rotation %i, LZ4_loadDict() : %i bytes", r, c); + + LZ4_initStream(dictStream, sizeof(*dictStream)); + LZ4_loadDict(dictStream, dict, (int)sizeof(dict)); + LZ4_initStream(stream, sizeof(*stream)); + LZ4_attach_dictionary(stream, dictStream); + c = LZ4_compress_fast_continue(stream, page + 8, dst, 4096, capacity, 1); + ASSERT_MSG(c > 0 && c <= 100, "rotation %i, LZ4_attach_dictionary() : %i bytes", r, c); + } +} + +/* the new block can overwrite the previous one (its dictionary) : any overlap must be handled */ +TEST(compress_continue_inputOverwritesPreviousBlock) +{ + /* block 1 : 1000 bytes at position 1000 ; block 2 : { position, size } */ + static const int overlaps[4][2] = { + { 500, 1000 }, /* covers the beginning of block 1 */ + { 1000, 1000 }, /* same place */ + { 500, 2000 }, /* covers block 1 entirely */ + { 1500, 1000 } }; /* covers the end of block 1 */ + int const pos1 = 1000, size1 = 1000; + int const capacity = LZ4_COMPRESSBOUND(4 KB); + char* const ring = (char*)UT_malloc(4 KB); + char* const src = (char*)UT_malloc(4 KB); /* content of both blocks */ + char* const cBlock1 = (char*)UT_malloc((size_t)capacity); + char* const cBlock2 = (char*)UT_malloc((size_t)capacity); + char* const decoded = (char*)UT_malloc(4 KB); + LZ4_stream_t* const stream = newStream(); + LZ4_streamHC_t* const streamHC = (LZ4_streamHC_t*)UT_defer(freeStreamHC, LZ4_createStreamHC()); + int ov, hc, trial; + ASSERT(streamHC != NULL); + memset(ring, 0, 4 KB); + for (ov = 0; ov < 4; ov++) + for (hc = 0; hc < 2; hc++) + for (trial = 0; trial < 64; trial++) { + int const pos2 = overlaps[ov][0], size2 = overlaps[ov][1]; + LZ4_streamDecode_t sd; + int c1, c2, i, r; + for (i = 0; i < size1 + size2; i++) src[i] = "ACGT"[UT_rand() & 3]; /* many matches */ + LZ4_initStream(stream, sizeof(*stream)); + LZ4_initStreamHC(streamHC, sizeof(*streamHC)); + memcpy(ring + pos1, src, (size_t)size1); + c1 = hc ? LZ4_compress_HC_continue(streamHC, ring + pos1, cBlock1, size1, capacity) + : LZ4_compress_fast_continue(stream, ring + pos1, cBlock1, size1, capacity, 1); + memcpy(ring + pos2, src + size1, (size_t)size2); + c2 = hc ? LZ4_compress_HC_continue(streamHC, ring + pos2, cBlock2, size2, capacity) + : LZ4_compress_fast_continue(stream, ring + pos2, cBlock2, size2, capacity, 1); + ASSERT(c1 > 0 && c2 > 0); + LZ4_setStreamDecode(&sd, NULL, 0); + r = LZ4_decompress_safe_continue(&sd, cBlock1, decoded, c1, size1); + r += LZ4_decompress_safe_continue(&sd, cBlock2, decoded + size1, c2, size2); + ASSERT_MSG(r == size1 + size2 && !memcmp(decoded, src, (size_t)(size1 + size2)), + "%s : corrupted output, block 2 (position %i, size %i) overwrites block 1 (position %i, size %i)", + hc ? "LZ4_compress_HC_continue()" : "LZ4_compress_fast_continue()", pos2, size2, pos1, size1); + } +} + + +static const UT_test tests[] = { + UT_TEST(decompress_safe_emptyInput), + UT_TEST(decompress_safe_oneByteInput), + UT_TEST(decompress_safe_shortcutEdgeCase), + UT_TEST(decompress_safe_all0xFF), + UT_TEST(decompress_safe_ipAtIend), + UT_TEST(decompress_safe_partial_offset0), + UT_TEST(compress_default_nullInput), + UT_TEST(compress_default_nullInputAndOutput), + UT_TEST(compress_destSize_extState_outOfRangeAcceleration), + UT_TEST(compress_default_inPlace), + UT_TEST(initStream_alignment), + UT_TEST(createStream), + UT_TEST(compress_fast_continue_roundTrip), + UT_TEST(resetStream_fast_largeFirstBlock), + UT_TEST(loadDict_tooSmall), + UT_TEST(saveDict_afterInit), + UT_TEST(saveDict_negativeSize), + UT_TEST(compress_periodicRuns), + UT_TEST(compress_continue_inputOverwritesPreviousBlock), +}; +UT_SUITE(lz4, tests); diff --git a/tests/unittest_lz4frame.c b/tests/unittest_lz4frame.c index 1df8b4ee..06d5d4d3 100644 --- a/tests/unittest_lz4frame.c +++ b/tests/unittest_lz4frame.c @@ -30,12 +30,6 @@ #include "lz4frame.h" #define LZ4_STATIC_LINKING_ONLY /* LZ4_ACCELERATION_MAX */ #include "lz4.h" -#include "datagen.h" /* RDG_genBuffer */ - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) - /*-************************************ * Helpers @@ -59,14 +53,6 @@ static void checkLZ4FError(size_t r, LZ4F_errorCodes code, LZ4F_isError(r) ? LZ4F_getErrorName(r) : "no error"); } -/* @return : @size bytes of compressible data, released at the end of the test */ -static void* sample(size_t size) -{ - void* const buffer = UT_malloc(size); - RDG_genBuffer(buffer, size, 0.5, 0.0, UT_rand()); - return buffer; -} - static void freeCctx(void* cctx) { LZ4F_freeCompressionContext((LZ4F_cctx*)cctx); } static void freeDctx(void* dctx) { LZ4F_freeDecompressionContext((LZ4F_dctx*)dctx); } static void freeCDict(void* cdict) { LZ4F_freeCDict((LZ4F_CDict*)cdict); } @@ -202,7 +188,7 @@ TEST(compressFrame_veryNegativeLevels) { static const int levels[] = { -LZ4_ACCELERATION_MAX /* cap - 1 */, INT_MIN + 1, INT_MIN }; size_t const srcSize = 2 MB; - const void* const src = sample(srcSize); + const void* const src = UT_sample(srcSize); size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL); void* const ref = UT_malloc(capacity); void* const frame = UT_malloc(capacity); @@ -220,7 +206,7 @@ TEST(compressFrame_veryNegativeLevels) TEST(compressFrame_validBlockSizeIDs) { size_t const srcSize = 256 KB; - const void* const src = sample(srcSize); + const void* const src = UT_sample(srcSize); size_t n; for (n = 0; n < ARRAY_SIZE(g_validBlockSizeIDs); n++) { LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES; @@ -250,7 +236,7 @@ TEST(compressBegin_invalidBlockSizeIDs) TEST(compressBegin_failurePreservesFrame) { size_t const srcSize = 256 KB; - const void* const src = sample(srcSize); + const void* const src = UT_sample(srcSize); LZ4F_cctx* const cctx = newCctx(); LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES; LZ4F_preferences_t invalidPrefs = LZ4F_INIT_PREFERENCES; @@ -276,7 +262,7 @@ TEST(compressBegin_afterFailedStateAllocation) LZ4F_CustomMem const failableMem = { failableMalloc, failableCalloc, failableFree, &failAlloc }; LZ4F_cctx* const cctx = (LZ4F_cctx*)UT_defer(freeCctx, LZ4F_createCompressionContext_advanced(failableMem, LZ4F_VERSION)); size_t const srcSize = 64 KB; - const void* const src = sample(srcSize); + const void* const src = UT_sample(srcSize); size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL); void* const frame = UT_malloc(capacity); LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES; @@ -294,7 +280,7 @@ TEST(compressBegin_afterFailedStateAllocation) TEST(compressEnd_contentSize) { size_t const srcSize = 2 MB; - const void* const src = sample(srcSize); + const void* const src = UT_sample(srcSize); LZ4F_cctx* const cctx = newCctx(); LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES; size_t const capacity = LZ4F_compressFrameBound(srcSize, &prefs); @@ -310,7 +296,7 @@ TEST(compressEnd_contentSize) TEST(compressEnd_wrongContentSize) { size_t const srcSize = 2 MB; - const void* const src = sample(srcSize); + const void* const src = UT_sample(srcSize); LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES; size_t const capacity = LZ4F_compressFrameBound(srcSize, &prefs); void* const frame = UT_malloc(capacity); @@ -322,7 +308,7 @@ TEST(compressBegin_usingDict) { size_t const dictSize = 7 KB; /* small enough for LZ4_MEMORY_USAGE == 10 */ size_t const srcSize = 66 KB; /* must be > 64 KB to avoid short-size optimizations */ - const char* const dict = (const char*)sample(dictSize + srcSize); + const char* const dict = (const char*)UT_sample(dictSize + srcSize); const char* const src = dict + dictSize; size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL); /* NULL preferences : 64 KB linked blocks */ void* const frame = UT_malloc(capacity); @@ -337,7 +323,7 @@ TEST(createCDict_advanced) int failAlloc = 1; LZ4F_CustomMem const failableMem = { failableMalloc, failableCalloc, failableFree, &failAlloc }; size_t const dictSize = 7 KB; - const void* const dict = sample(dictSize); + const void* const dict = UT_sample(dictSize); ASSERT(UT_defer(freeCDict, LZ4F_createCDict_advanced(failableMem, dict, dictSize)) == NULL); /* uses failableMem */ failAlloc = 0; ASSERT(UT_defer(freeCDict, LZ4F_createCDict_advanced(failableMem, dict, dictSize)) != NULL); @@ -355,7 +341,7 @@ static LZ4F_CDict* newCDict(const void* dict) TEST(compressFrame_usingCDict) { size_t const srcSize = 65 KB; /* must be > 64 KB to avoid short-size optimizations */ - const void* const src = sample(srcSize); + const void* const src = UT_sample(srcSize); LZ4F_CDict* const cdict = newCDict(src); size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL); void* const frame = UT_malloc(capacity); @@ -369,7 +355,7 @@ TEST(compressFrame_usingCDict_extremeLevels) { int const levels[] = { -3, LZ4F_compressionLevel_max() }; size_t const srcSize = CDICT_DICT_SIZE; - const void* const src = sample(srcSize); + const void* const src = UT_sample(srcSize); LZ4F_CDict* const cdict = newCDict(src); LZ4F_cctx* const cctx = newCctx(); size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL); @@ -387,7 +373,7 @@ TEST(compressFrame_usingCDict_extremeLevels) TEST(compressFrame_usingCDict_blockModes) { size_t const srcSize = 3 * 64 KB; /* a single block would always be independent */ - const void* const src = sample(srcSize); + const void* const src = UT_sample(srcSize); LZ4F_CDict* const cdict = newCDict(src); LZ4F_cctx* const cctx = newCctx(); size_t const capacity = LZ4F_compressFrameBound(srcSize, NULL); diff --git a/tests/unittest_lz4hc.c b/tests/unittest_lz4hc.c new file mode 100644 index 00000000..148e31c2 --- /dev/null +++ b/tests/unittest_lz4hc.c @@ -0,0 +1,374 @@ +/* + unittest_lz4hc - unit tests of lz4hc (block format, high compression levels) + Copyright (C) Yann Collet 2026 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +#include "unittest.h" +#include /* memset, memcpy, memcmp */ +#define LZ4_DISABLE_DEPRECATE_WARNINGS /* LZ4_createHC, LZ4_compressHC2_continue */ +#define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY /* LZ4_favorDecompressionSpeed */ +#include "lz4hc.h" + + +/*-************************************ +* Helpers +**************************************/ +/* the default level, and the first level of the optimal parser */ +static const int g_levels[] = { LZ4HC_CLEVEL_DEFAULT, LZ4HC_CLEVEL_OPT_MIN }; + +static void freeStreamHC(void* stream) { LZ4_freeStreamHC((LZ4_streamHC_t*)stream); } + +/* stream, released at the end of the test */ +static LZ4_streamHC_t* newStreamHC(int level) +{ + LZ4_streamHC_t* const stream = (LZ4_streamHC_t*)UT_defer(freeStreamHC, LZ4_createStreamHC()); + ASSERT(stream != NULL); + LZ4_resetStreamHC_fast(stream, level); + return stream; +} + +/* a successful compression leaves the stream clean */ +#define ASSERT_CLEAN(stream) ASSERT(!(stream)->internal_donotuse.dirty) + + +/*-************************************ +* Tests +**************************************/ +/* to be tested with UBSan */ +TEST(compress_HC_nullInput) +{ + int level; + for (level = LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++) { + char dst[16]; + int const cSize = LZ4_compress_HC(NULL, dst, 0, LZ4_compressBound(0), level); + ASSERT_MSG(cSize == 1 && dst[0] == 0, "level %i : compressing empty input should give byte 0", level); + } +} + +TEST(compress_HC_nullInputAndOutput) +{ + int level; + for (level = LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++) + ASSERT_MSG(LZ4_compress_HC(NULL, NULL, 0, 0, level) == 0, "level %i : compressing into NULL must fail", level); +} + +TEST(initStreamHC_alignment) +{ + typedef struct { + LZ4_streamHC_t hc1; + LZ4_streamHC_t hc2; + char c; + LZ4_streamHC_t hc3; + } shct; + shct* const shc = (shct*)UT_malloc(sizeof(shct)); + memset(shc, 0, sizeof(*shc)); + ASSERT(LZ4_initStreamHC(&shc->hc1, sizeof(shc->hc1)) != NULL); + ASSERT(LZ4_initStreamHC(&shc->hc2, sizeof(shc->hc2)) != NULL); + ASSERT(LZ4_initStreamHC(&shc->hc3, sizeof(shc->hc3)) != NULL); + ASSERT(LZ4_initStreamHC((char*)&shc->hc1 + 1, sizeof(shc->hc1)) == NULL); /* bad alignment */ +} + +TEST(createStreamHC) +{ + newStreamHC(LZ4HC_CLEVEL_DEFAULT); +} + +TEST(compress_HC_continue_roundTrip) +{ + int const srcSize = 130 KB; + const char* const src = (const char*)UT_sample((size_t)srcSize); + char* const dst = (char*)UT_malloc((size_t)srcSize); + char* const decoded = (char*)UT_malloc((size_t)srcSize); + LZ4_streamHC_t* const stream = (LZ4_streamHC_t*)UT_malloc(sizeof(LZ4_streamHC_t)); /* statically allocated */ + size_t l; + for (l = 0; l < ARRAY_SIZE(g_levels); l++) { + int cSize; + LZ4_initStreamHC(stream, sizeof(*stream)); + LZ4_setCompressionLevel(stream, g_levels[l]); + cSize = LZ4_compress_HC_continue(stream, src, dst, srcSize, srcSize - 1); + ASSERT(cSize > 0); + ASSERT_CLEAN(stream); + ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, srcSize), srcSize); + ASSERT_MEMEQ(decoded, src, (size_t)srcSize); + } +} + +/* deprecated streaming API */ +TEST(compressHC2_continue_roundTrip) +{ + int const srcSize = 64 KB; + const char* const src = (const char*)UT_sample((size_t)srcSize); + char* const dst = (char*)UT_malloc((size_t)LZ4_compressBound(srcSize)); + char* const decoded = (char*)UT_malloc((size_t)srcSize); + size_t l; + for (l = 0; l < ARRAY_SIZE(g_levels); l++) { + void* const ctx = LZ4_createHC(src); + int cSize; + ASSERT(ctx != NULL); + cSize = LZ4_compressHC2_continue(ctx, src, dst, srcSize, g_levels[l]); + LZ4_freeHC(ctx); + ASSERT(cSize > 0); + ASSERT_EQ(LZ4_decompress_safe(dst, decoded, cSize, srcSize), srcSize); + ASSERT_MEMEQ(decoded, src, (size_t)srcSize); + } +} + +/* level 2 : the match tested at ip+1 must not be ip itself, inside runs */ +TEST(compress_HC_level2_runs) +{ + char runs[2 + 4096]; /* 0xFF 0x00, then runs of 10 x 0xFF and 10 x 0x00 */ + int const capacity = LZ4_COMPRESSBOUND(sizeof(runs)); + char* const dst = (char*)UT_malloc((size_t)capacity); + int i, cSize; + runs[0] = (char)0xFF; runs[1] = 0; + for (i = 0; i < 4096; i++) runs[2 + i] = (i % 20) < 10 ? (char)0xFF : 0; + cSize = LZ4_compress_HC(runs, dst, (int)sizeof(runs), capacity, 2); + /* ~1.6 KB (one literal and one offset 1 per run), instead of < 50 bytes */ + ASSERT_MSG(cSize > 0 && cSize <= 100, "%i bytes", cSize); +} + +/* #926 */ +TEST(saveDictHC_beforeCompression) +{ + /* first : level of LZ4_initStreamHC() */ + static const int levels[] = { LZ4HC_CLEVEL_DEFAULT, 3, LZ4HC_CLEVEL_OPT_MIN, LZ4HC_CLEVEL_MAX }; + const char* const dict = (const char*)UT_sample(128 KB); + const char* const src = dict + 64 KB; + int const capacity = LZ4_COMPRESSBOUND(64 KB); + char* const dst = (char*)UT_malloc((size_t)capacity); + char* const decoded = (char*)UT_malloc(64 KB); + LZ4_streamHC_t* const stream = (LZ4_streamHC_t*)UT_malloc(sizeof(LZ4_streamHC_t)); + LZ4_streamHC_t* const dictStream = newStreamHC(LZ4HC_CLEVEL_DEFAULT); + char saveBuffer[240]; + size_t l; + int attach; + ASSERT(LZ4_initStreamHC(stream, sizeof(*stream)) != NULL); + ASSERT_EQ(LZ4_saveDictHC(stream, NULL, 0), 0); + ASSERT_EQ(LZ4_saveDictHC(stream, saveBuffer, sizeof(saveBuffer)), 0); /* compression hasn't started */ + + /* next block is compressed as the first one of the stream, + * without (attach==0) or with (attach==1) an attached dictionary */ + for (l = 0; l < ARRAY_SIZE(levels); l++) { + for (attach = 0; attach <= 1; attach++) { + int cSize; + if (l || attach) LZ4_resetStreamHC_fast(stream, levels[l]); /* first test : right after init */ + if (attach) { + LZ4_resetStreamHC_fast(dictStream, levels[l]); + ASSERT_EQ(LZ4_loadDictHC(dictStream, dict, 64 KB), 64 KB); + LZ4_attach_HC_dictionary(stream, dictStream); + } + ASSERT_EQ(LZ4_saveDictHC(stream, saveBuffer, sizeof(saveBuffer)), 0); + cSize = LZ4_compress_HC_continue(stream, src, dst, 64 KB, capacity); + ASSERT(cSize > 0); + ASSERT_MSG(LZ4_decompress_safe_usingDict(dst, decoded, cSize, 64 KB, dict, attach ? 64 KB : 0) == 64 KB + && !memcmp(decoded, src, 64 KB), + "level %i, dictionary %i : block compressed after LZ4_saveDictHC() is corrupted", levels[l], attach); + } } +} + +TEST(compress_HC_destSize_longSequence) +{ + size_t const srcSize = 1 MB; + size_t const targetSize = 4116; /* size carefully selected to trigger an overflow */ + char* const src = (char*)UT_malloc(srcSize); + char* const dst = (char*)UT_malloc(targetSize + 1); + char const sentinel = 101; + LZ4_streamHC_t* const stream = newStreamHC(3); + int readSize = (int)srcSize; + int cSize; + memset(src, 0, srcSize); + dst[targetSize] = sentinel; + cSize = LZ4_compress_HC_destSize(stream, src, dst, &readSize, (int)targetSize, 3); + ASSERT_EQ(cSize, targetSize); /* fills dst exactly */ + ASSERT_EQ(dst[targetSize], sentinel); + ASSERT_MSG(readSize >= 1045000, "doesn't compress enough (%i -> %i)", readSize, cSize); +} + +/* a failed compression marks the state as dirty, which is fully re-initialized at next reset */ +TEST(favorDecompressionSpeed_afterFailedCompression) +{ + int const srcSize = 64 KB; + int const capacity = LZ4_COMPRESSBOUND(srcSize); + int const level = LZ4HC_CLEVEL_OPT_MIN; + const char* const src = (const char*)UT_sample((size_t)srcSize); + char* const ref = (char*)UT_malloc((size_t)capacity); + char* const dst = (char*)UT_malloc((size_t)capacity); + LZ4_streamHC_t* const stream = newStreamHC(level); + int refSize, cSize; + + /* reference : favor decompression speed */ + LZ4_favorDecompressionSpeed(stream, 1); + refSize = LZ4_compress_HC_continue(stream, src, ref, srcSize, capacity); + ASSERT(refSize > 0); + + /* this test is only meaningful if favorDecSpeed changes the output */ + LZ4_resetStreamHC_fast(stream, level); + LZ4_favorDecompressionSpeed(stream, 0); + cSize = LZ4_compress_HC_continue(stream, src, dst, srcSize, capacity); + ASSERT(cSize != refSize || memcmp(dst, ref, (size_t)cSize)); + + /* failed compression */ + LZ4_resetStreamHC_fast(stream, level); + LZ4_favorDecompressionSpeed(stream, 1); + ASSERT_EQ(LZ4_compress_HC_continue(stream, src, dst, srcSize, refSize - 1), 0); + + /* next stream : must still favor decompression speed */ + LZ4_resetStreamHC_fast(stream, level); + ASSERT_EQ(LZ4_compress_HC_continue(stream, src, dst, srcSize, capacity), refSize); + ASSERT_MEMEQ(dst, ref, (size_t)refSize); +} + +TEST(compress_HC_continue_dictionary) +{ + int const dictSize = 64 KB; + int const srcSize = 130 KB; + const char* const dict = (const char*)UT_sample((size_t)(dictSize + srcSize)); + const char* const src = dict + dictSize; + char* const dst = (char*)UT_malloc((size_t)srcSize); + char* const decoded = (char*)UT_malloc((size_t)srcSize); + size_t l; + for (l = 0; l < ARRAY_SIZE(g_levels); l++) { + LZ4_streamHC_t* const stream = newStreamHC(g_levels[l]); + int cSize; + LZ4_loadDictHC(stream, dict, dictSize); + cSize = LZ4_compress_HC_continue(stream, src, dst, srcSize, srcSize - 1); + ASSERT(cSize > 0); + ASSERT_CLEAN(stream); + ASSERT_EQ(LZ4_decompress_safe_usingDict(dst, decoded, cSize, srcSize, dict, dictSize), srcSize); + ASSERT_MEMEQ(decoded, src, (size_t)srcSize); + } +} + +/* 2 blocks : the 2nd one also references the 1st one */ +TEST(compress_HC_continue_dictionary_twoBlocks) +{ + int const segSize = 65 KB; + const char* const input = (const char*)UT_sample((size_t)(3 * segSize)); /* dictionary, block 1, block 2 */ + char* const dst = (char*)UT_malloc((size_t)(2 * segSize)); + char* const decoded = (char*)UT_malloc((size_t)(2 * segSize)); + size_t l; + for (l = 0; l < ARRAY_SIZE(g_levels); l++) { + LZ4_streamHC_t* const stream = newStreamHC(g_levels[l]); + int c1, c2; + LZ4_loadDictHC(stream, input, segSize); + c1 = LZ4_compress_HC_continue(stream, input + segSize, dst, segSize, segSize - 1); + ASSERT(c1 > 0); + ASSERT_CLEAN(stream); + c2 = LZ4_compress_HC_continue(stream, input + 2 * segSize, dst + c1, segSize, segSize - 1); + ASSERT(c2 > 0); + ASSERT_CLEAN(stream); + ASSERT_EQ(LZ4_decompress_safe_usingDict(dst, decoded, c1, segSize, input, segSize), segSize); + ASSERT_EQ(LZ4_decompress_safe_usingDict(dst + c1, decoded + segSize, c2, segSize, input, 2 * segSize), segSize); + ASSERT_MEMEQ(decoded, input + segSize, (size_t)(2 * segSize)); + } +} + +/* the dictionary doesn't immediately precede the input */ +TEST(compress_HC_continue_remoteDictionary) +{ + int const dictSize = 32 KB; + int const srcSize = 130 KB; + const char* const dict = (const char*)UT_sample((size_t)(64 KB + srcSize)); + const char* const src = dict + 64 KB; + char* const dst = (char*)UT_malloc((size_t)srcSize); + char* const decoded = (char*)UT_malloc((size_t)srcSize); + size_t l; + for (l = 0; l < ARRAY_SIZE(g_levels); l++) { + LZ4_streamHC_t* const stream = newStreamHC(g_levels[l]); + int cSize; + LZ4_loadDictHC(stream, dict, dictSize); + cSize = LZ4_compress_HC_continue(stream, src, dst, srcSize, srcSize - 1); + ASSERT(cSize > 0); + ASSERT_CLEAN(stream); + ASSERT_EQ(LZ4_decompress_safe_usingDict(dst, decoded, cSize, srcSize, dict, dictSize), srcSize); + ASSERT_MEMEQ(decoded, src, (size_t)srcSize); + } +} + +/* on long sequences of zeros, LZ4_compress_HC_destSize() and LZ4_compress_destSize() produce the same output */ +TEST(compress_HC_destSize_sameAsFast) +{ + int const srcCapacity = 3 MB; + int const dstCapacity = 6 KB; + int const dstStep = 43; + int const dstMinLen = 33 + (int)(UT_rand() % (unsigned)dstStep); + int const dstMaxLen = 5000; + char const endMark = (char)0x88; + char* const src1 = (char*)UT_malloc((size_t)srcCapacity + 1); + char* const src2 = (char*)UT_malloc((size_t)srcCapacity + 1); + char* const dst1 = (char*)UT_malloc((size_t)dstCapacity + 1); + char* const dst2 = (char*)UT_malloc((size_t)dstCapacity + 1); + LZ4_streamHC_t* const stream = newStreamHC(LZ4HC_CLEVEL_DEFAULT); + size_t l; + int dlen, slen; + ASSERT(dstCapacity > dstMaxLen); + for (l = 0; l < ARRAY_SIZE(g_levels); l++) + for (dlen = dstMinLen; dlen <= dstMaxLen; dlen += dstStep) { + int const srcLen = (dlen - 10) * 255 + 24; + if (srcLen + 10 >= srcCapacity) break; + for (slen = srcLen - 3; slen <= srcLen + 3; slen++) { + int srcSize1 = slen, srcSize2 = slen; + int dSize1, dSize2; + memset(src1, 0, (size_t)slen); + memset(dst1, 0, (size_t)dlen); + dst1[dlen] = endMark; + dSize1 = LZ4_compress_destSize(src1, dst1, &srcSize1, dlen); + ASSERT_EQ(dst1[dlen], endMark); /* no overwrite beyond dst */ + ASSERT(dSize1 > 0 && dSize1 <= dlen); + ASSERT(srcSize1 <= slen); + ASSERT_EQ(LZ4_decompress_safe(dst1, src1, dSize1, srcCapacity), srcSize1); + + memset(src2, 0, (size_t)slen); + memset(dst2, 0, (size_t)dlen); + dst2[dlen] = endMark; + LZ4_resetStreamHC(stream, g_levels[l]); + dSize2 = LZ4_compress_HC_destSize(stream, src2, dst2, &srcSize2, dlen, g_levels[l]); + ASSERT_EQ(dst2[dlen], endMark); + ASSERT_EQ(dSize2, dSize1); + ASSERT_EQ(srcSize2, srcSize1); + ASSERT_MEMEQ(dst2, dst1, (size_t)dSize2); + ASSERT_EQ(LZ4_decompress_safe(dst2, src1, dSize2, srcCapacity), srcSize1); + ASSERT_MEMEQ(src1, src2, (size_t)srcSize1); + } } +} + + +static const UT_test tests[] = { + UT_TEST(compress_HC_nullInput), + UT_TEST(compress_HC_nullInputAndOutput), + UT_TEST(initStreamHC_alignment), + UT_TEST(createStreamHC), + UT_TEST(compress_HC_continue_roundTrip), + UT_TEST(compressHC2_continue_roundTrip), + UT_TEST(compress_HC_level2_runs), + UT_TEST(saveDictHC_beforeCompression), + UT_TEST(compress_HC_destSize_longSequence), + UT_TEST(favorDecompressionSpeed_afterFailedCompression), + UT_TEST(compress_HC_continue_dictionary), + UT_TEST(compress_HC_continue_dictionary_twoBlocks), + UT_TEST(compress_HC_continue_remoteDictionary), + UT_TEST(compress_HC_destSize_sameAsFast), +}; +UT_SUITE(lz4hc, tests);