From db0b5d7d1e1bc884688b77a0eb4f66005fad3c39 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 27 Oct 2020 16:57:24 -0400 Subject: [PATCH 1/4] Add test to fuzzer.c --- tests/fuzzer.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 7d1d0153d..f0ac24677 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -385,6 +385,42 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : LDM + opt parser with uncompressible block ", testNb++); + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t const srcSize = 300 KB; + size_t const flushSize = 128 KB + 5; + size_t const dstSize = ZSTD_compressBound(srcSize); + char* src = CNBuffer; + char* dst = compressedBuffer; + + ZSTD_outBuffer out = { dst, dstSize, 0 }; + ZSTD_inBuffer in = { src, flushSize, 0 }; + + RDG_genBuffer(src, srcSize, 0.5, 0.5, seed); + /* Force an LDM to exist that crosses block boundary into uncompressible block */ + memcpy(src + 125 KB, src, 3 KB + 5); + + /* Enable MT, LDM, and opt parser */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); + + /* Flushes a block of 128 KB and block of 5 bytes */ + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + + /* Compress the rest */ + in.size = 300 KB; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos)); + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); if (cctx==NULL) goto _output_error; From 169fc07aa14a35e259691c79545c2bd77810b764 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 27 Oct 2020 16:59:43 -0400 Subject: [PATCH 2/4] Move test to appropriate location --- tests/fuzzer.c | 72 +++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index f0ac24677..eb1b28c1c 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -385,42 +385,6 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3i : LDM + opt parser with uncompressible block ", testNb++); - { ZSTD_CCtx* cctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); - size_t const srcSize = 300 KB; - size_t const flushSize = 128 KB + 5; - size_t const dstSize = ZSTD_compressBound(srcSize); - char* src = CNBuffer; - char* dst = compressedBuffer; - - ZSTD_outBuffer out = { dst, dstSize, 0 }; - ZSTD_inBuffer in = { src, flushSize, 0 }; - - RDG_genBuffer(src, srcSize, 0.5, 0.5, seed); - /* Force an LDM to exist that crosses block boundary into uncompressible block */ - memcpy(src + 125 KB, src, 3 KB + 5); - - /* Enable MT, LDM, and opt parser */ - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); - CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); - - /* Flushes a block of 128 KB and block of 5 bytes */ - CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); - - /* Compress the rest */ - in.size = 300 KB; - CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); - - CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos)); - - ZSTD_freeCCtx(cctx); - ZSTD_freeDCtx(dctx); - } - DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); if (cctx==NULL) goto _output_error; @@ -754,6 +718,42 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t const srcSize = 300 KB; + size_t const flushSize = 128 KB + 5; + size_t const dstSize = ZSTD_compressBound(srcSize); + char* src = CNBuffer; + char* dst = compressedBuffer; + + ZSTD_outBuffer out = { dst, dstSize, 0 }; + ZSTD_inBuffer in = { src, flushSize, 0 }; + + RDG_genBuffer(src, srcSize, 0.5, 0.5, seed); + /* Force an LDM to exist that crosses block boundary into uncompressible block */ + memcpy(src + 125 KB, src, 3 KB + 5); + + /* Enable MT, LDM, and opt parser */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); + + /* Flushes a block of 128 KB and block of 5 bytes */ + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + + /* Compress the rest */ + in.size = 300 KB; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + + CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos)); + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); From 60a52c29e623c71aba56d98c39360425b48d4c12 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 28 Oct 2020 16:22:22 -0400 Subject: [PATCH 3/4] Add check for allocation --- tests/fuzzer.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index eb1b28c1c..c2f5d5707 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -721,6 +721,11 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); { ZSTD_CCtx* cctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); + if (!cctx || !dctx) { + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } size_t const srcSize = 300 KB; size_t const flushSize = 128 KB + 5; size_t const dstSize = ZSTD_compressBound(srcSize); From 7198ebb21301a8665cd36db46a3c049ba1d4026b Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Wed, 28 Oct 2020 17:49:54 -0400 Subject: [PATCH 4/4] Un-mix declarations and code --- tests/fuzzer.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index c2f5d5707..1297b61b5 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -721,19 +721,20 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); { ZSTD_CCtx* cctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t const srcSize = 300 KB; + size_t const flushSize = 128 KB + 5; + size_t const dstSize = ZSTD_compressBound(srcSize); + char* src = (char*)CNBuffer; + char* dst = (char*)compressedBuffer; + + ZSTD_outBuffer out = { dst, dstSize, 0 }; + ZSTD_inBuffer in = { src, flushSize, 0 }; + if (!cctx || !dctx) { DISPLAY("Not enough memory, aborting\n"); testResult = 1; goto _end; } - size_t const srcSize = 300 KB; - size_t const flushSize = 128 KB + 5; - size_t const dstSize = ZSTD_compressBound(srcSize); - char* src = CNBuffer; - char* dst = compressedBuffer; - - ZSTD_outBuffer out = { dst, dstSize, 0 }; - ZSTD_inBuffer in = { src, flushSize, 0 }; RDG_genBuffer(src, srcSize, 0.5, 0.5, seed); /* Force an LDM to exist that crosses block boundary into uncompressible block */