diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index 341877858..356549f36 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -575,6 +575,8 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, BYTE const* const iend = istart + srcSize; /* Input positions */ BYTE const* ip = istart; + + /* If using opt parser, use LDMs only as candidates rather than always accepting them */ if (cParams->strategy >= ZSTD_btopt) { size_t lastLLSize; ms->ldmSeqStore = *rawSeqStore; diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index fa3587bed..5211fc917 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -774,7 +774,9 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( static void ZSTD_opt_skipBytesInLdmSeqStore(rawSeqStore_t* ldmSeqStore, size_t nbBytes) { while (nbBytes && ldmSeqStore->pos < ldmSeqStore->size) { rawSeq currSeq = ldmSeqStore->seq[ldmSeqStore->pos]; + /* posInSequence necessarily must never represent a value beyond the sequence */ assert(ldmSeqStore->posInSequence <= currSeq.matchLength + currSeq.litLength); + if (nbBytes <= currSeq.litLength) { ldmSeqStore->posInSequence += nbBytes; return; @@ -782,6 +784,7 @@ static void ZSTD_opt_skipBytesInLdmSeqStore(rawSeqStore_t* ldmSeqStore, size_t n ldmSeqStore->posInSequence += currSeq.litLength; nbBytes -= currSeq.litLength; } + if (nbBytes < currSeq.matchLength) { ldmSeqStore->posInSequence += nbBytes; return; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 8b10078ac..82d2f119e 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -780,6 +780,44 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing ldm no regressions in size for opt parser : ", testNb++); + { + size_t cSizeLdm; + size_t cSizeNoLdm; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + + RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed); + + /* Enable checksum to verify round trip. */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); + + /* Round trip once with ldm. */ + cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(cSizeLdm); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm)); + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); + + /* Round trip once without ldm. */ + cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(cSizeNoLdm); + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm)); + + if (cSizeLdm > cSizeNoLdm) { + DISPLAY("Using long mode should not cause regressions for btopt+\n"); + testResult = 1; + goto _output_error; + } + + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + /* Note: this test takes 0.5 seconds to run */ DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++); { diff --git a/tests/playTests.sh b/tests/playTests.sh index 3959a8f3a..49371040e 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1217,7 +1217,7 @@ then println "\n===> zstdmt round-trip tests " roundTripTest -g4M "1 -T0" roundTripTest -g8M "3 -T2" - roundTripTest -g8M "3 -T0 --long" + roundTripTest -g8M "19 -T0 --long" roundTripTest -g8000K "2 --threads=2" fileRoundTripTest -g4M "19 -T2 -B1M"