From bde926fce7b9bf1e7c4688569819c3c0bd2facd2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 18 May 2016 17:18:48 +0200 Subject: [PATCH] removed msan tests --- .travis.yml | 1 - Makefile | 2 +- programs/datagen.c | 102 ++++++++++++++++++++++--------------------- programs/zbufftest.c | 22 +++++----- 4 files changed, 64 insertions(+), 63 deletions(-) diff --git a/.travis.yml b/.travis.yml index a430fee56..b7faed3eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,6 @@ env: - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - ZSTD_TRAVIS_CI_ENV=usan - ZSTD_TRAVIS_CI_ENV=asan - - ZSTD_TRAVIS_CI_ENV=msan - ZSTD_TRAVIS_CI_ENV=asan32 - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" diff --git a/Makefile b/Makefile index b1b9096a9..e7fca85fe 100644 --- a/Makefile +++ b/Makefile @@ -139,7 +139,7 @@ asan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" msan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason asan32: clean $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" diff --git a/programs/datagen.c b/programs/datagen.c index ecc783e1b..a17debcf0 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -48,22 +48,25 @@ * Macros **************************************/ #define KB *(1 <<10) +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +#define RDG_DEBUG 0 +#define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ ) /*-************************************ -* Local types +* Local constants **************************************/ #define LTLOG 13 #define LTSIZE (1<> (32 - r))) -static unsigned int RDG_rand(U32* src) +static U32 RDG_rand(U32* src) { static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; @@ -72,39 +75,42 @@ static unsigned int RDG_rand(U32* src) rand32 ^= prime2; rand32 = RDG_rotl32(rand32, 13); *src = rand32; - return rand32; + return rand32 >> 5; } -static void RDG_fillLiteralDistrib(litDistribTable lt, double ld) +static void RDG_fillLiteralDistrib(BYTE* ldt, double ld) { - U32 i = 0; + BYTE const firstChar = (ld<=0.0) ? 0 : '('; + BYTE const lastChar = (ld<=0.0) ? 255 : '}'; BYTE character = (ld<=0.0) ? 0 : '0'; - BYTE const firstChar = (ld<=0.0) ? 0 : '('; - BYTE const lastChar = (ld<=0.0) ?255: '}'; + U32 u; - while (i LTSIZE) weight = LTSIZE-i; - end = i + weight; - while (i < end) lt[i++] = character; + if (ld<=0.0) ld = 0.0; + //TRACE(" percent:%5.2f%% \n", ld*100.); + //TRACE(" start:(%c)[%02X] ", character, character); + for (u=0; u lastChar) character = firstChar; } } -static BYTE RDG_genChar(U32* seed, const litDistribTable lt) +static BYTE RDG_genChar(U32* seed, const BYTE* ldt) { U32 const id = RDG_rand(seed) & LTMASK; - return (lt[id]); + //TRACE(" %u : \n", id); + //TRACE(" %4u [%4u] ; val : %4u \n", id, id&255, ldt[id]); + return (ldt[id]); /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with 0.0. Checked : table is fully initialized */ } -#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 0x7FFF) -#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) -void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr) +#define RDG_RAND15BITS ( RDG_rand(seed) & 0x7FFF ) +#define RDG_RANDLENGTH ( (RDG_rand(seed) & 7) ? (RDG_rand(seed) & 0xF) : (RDG_rand(seed) & 0x1FF) + 0xF) +void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr) { BYTE* buffPtr = (BYTE*)buffer; const U32 matchProba32 = (U32)(32768 * matchProba); @@ -123,75 +129,73 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match } memset(buffPtr+pos, 0, size0); pos += size0; - buffPtr[pos-1] = RDG_genChar(seed, lt); + buffPtr[pos-1] = RDG_genChar(seed, ldt); continue; } /* init */ - if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1; + if (pos==0) buffPtr[0] = RDG_genChar(seed, ldt), pos=1; /* Generate compressible data */ while (pos < buffSize) { /* Select : Literal (char) or Match (within 32K) */ if (RDG_RAND15BITS < matchProba32) { /* Copy (within 32K) */ - size_t match; - size_t d; - size_t const length = RDG_RANDLENGTH + 4; - U32 offset = RDG_RAND15BITS + 1; - U32 repeatOffset = (RDG_rand(seed) & 15) == 2; - if (repeatOffset) offset = prevOffset; - if (offset > pos) offset = (U32)pos; - prevOffset = offset; - match = pos - offset; - d = pos + length; - if (d > buffSize) d = buffSize; + U32 const length = RDG_RANDLENGTH + 4; + U32 const d = MIN (pos + length , buffSize); + U32 const repeatOffset = (RDG_rand(seed) & 15) == 2; + U32 const randOffset = RDG_RAND15BITS + 1; + U32 const offset = repeatOffset ? prevOffset : MIN(randOffset , pos); + size_t match = pos - offset; + //TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length); while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ + prevOffset = offset; } else { /* Literal (noise) */ - size_t const length = RDG_RANDLENGTH; - size_t d = pos + length; - if (d > buffSize) d = buffSize; - while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt); + U32 const length = RDG_RANDLENGTH; + U32 const d = MIN(pos + length, buffSize); + while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldt); } } } void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) { - litDistribTable lt; - if (litProba==0.0) litProba = matchProba / 4.5; - RDG_fillLiteralDistrib(lt, litProba); - RDG_genBlock(buffer, size, 0, matchProba, lt, &seed); + BYTE ldt[LTSIZE]; + memset(ldt, '0', sizeof(ldt)); + if (litProba<=0.0) litProba = matchProba / 4.5; + //TRACE(" percent:%5.2f%% \n", litProba*100.); + RDG_fillLiteralDistrib(ldt, litProba); + RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed); } -#define RDG_DICTSIZE (32 KB) -#define RDG_BLOCKSIZE (128 KB) -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed) { - BYTE* buff = (BYTE*)malloc(RDG_DICTSIZE + RDG_BLOCKSIZE); + size_t const stdBlockSize = 128 KB; + size_t const stdDictSize = 32 KB; + BYTE* buff = (BYTE*)malloc(stdDictSize + stdBlockSize); U64 total = 0; - litDistribTable ldt; + BYTE ldt[LTSIZE]; /* init */ if (buff==NULL) { fprintf(stdout, "not enough memory\n"); exit(1); } if (litProba<=0.0) litProba = matchProba / 4.5; + memset(ldt, '0', sizeof(ldt)); RDG_fillLiteralDistrib(ldt, litProba); SET_BINARY_MODE(stdout); /* Generate initial dict */ - RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldt, &seed); + RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed); /* Generate compressible data */ while (total < size) { - size_t const genBlockSize = (size_t) (MIN (RDG_BLOCKSIZE, size-total)); - RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldt, &seed); + size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total)); + RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed); total += genBlockSize; { size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; } /* update dict */ - memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); + memcpy(buff, buff + stdBlockSize, stdDictSize); } /* cleanup */ diff --git a/programs/zbufftest.c b/programs/zbufftest.c index f8d527ef2..eb63117ab 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -140,7 +140,6 @@ static int basicUnitTests(U32 seed, double compressibility) void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); - U32 randState = seed; size_t result, cSize, readSize, genSize; U32 testNb=0; ZBUFF_CCtx* zc = ZBUFF_createCCtx(); @@ -151,7 +150,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAY("Not enough memory, aborting\n"); goto _output_error; } - RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., randState); + RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); @@ -234,19 +233,13 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max) { const BYTE* b1 = (const BYTE*)buf1; const BYTE* b2 = (const BYTE*)buf2; - size_t i; - for (i=0; i "); DISPLAY(__VA_ARGS__); \ - DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } - - static size_t FUZ_rLogLength(U32* seed, U32 logLength) { size_t const lengthMask = ((size_t)1 << logLength) - 1; @@ -259,6 +252,11 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog) return FUZ_rLogLength(seed, logLength); } +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { static const U32 maxSrcLog = 24;