lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-07 06:23:00 +03:00
Code Activity

removed msan tests

Browse Source
This commit is contained in:
Yann Collet
2016-05-18 17:18:48 +02:00
parent d9967436da
commit bde926fce7
4 changed files with 64 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.travis.yml
View File

@@ -19,7 +19,6 @@ env:
- ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy" - ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy"
- ZSTD_TRAVIS_CI_ENV=usan - ZSTD_TRAVIS_CI_ENV=usan
- ZSTD_TRAVIS_CI_ENV=asan - ZSTD_TRAVIS_CI_ENV=asan
- ZSTD_TRAVIS_CI_ENV=msan
- ZSTD_TRAVIS_CI_ENV=asan32 - ZSTD_TRAVIS_CI_ENV=asan32
- ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest" - ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest"

2
Makefile
View File

@@ -139,7 +139,7 @@ asan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
msan: clean msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason
asan32: clean asan32: clean
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" $(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"

102
programs/datagen.c
View File

@@ -48,22 +48,25 @@
* Macros * Macros
**************************************/ **************************************/
#define KB *(1 <<10) #define KB *(1 <<10)
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define RDG_DEBUG 0
#define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ )
/*-************************************ /*-************************************
* Local types * Local constants
**************************************/ **************************************/
#define LTLOG 13 #define LTLOG 13
#define LTSIZE (1<<LTLOG) #define LTSIZE (1<<LTLOG)
#define LTMASK (LTSIZE-1) #define LTMASK (LTSIZE-1)
typedef BYTE litDistribTable[LTSIZE];
/*-******************************************************* /*-*******************************************************
* Local Functions * Local Functions
*********************************************************/ *********************************************************/
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) #define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned int RDG_rand(U32* src) static U32 RDG_rand(U32* src)
{ {
static const U32 prime1 = 2654435761U; static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U; static const U32 prime2 = 2246822519U;
@@ -72,39 +75,42 @@ static unsigned int RDG_rand(U32* src)
rand32 ^= prime2; rand32 ^= prime2;
rand32 = RDG_rotl32(rand32, 13); rand32 = RDG_rotl32(rand32, 13);
*src = rand32; *src = rand32;
return rand32; return rand32 >> 5;
} }
static void RDG_fillLiteralDistrib(litDistribTable lt, double ld) static void RDG_fillLiteralDistrib(BYTE* ldt, double ld)
{ {
U32 i = 0; BYTE const firstChar = (ld<=0.0) ? 0 : '(';
BYTE const lastChar = (ld<=0.0) ? 255 : '}';
BYTE character = (ld<=0.0) ? 0 : '0'; BYTE character = (ld<=0.0) ? 0 : '0';
BYTE const firstChar = (ld<=0.0) ? 0 : '('; U32 u;
BYTE const lastChar = (ld<=0.0) ?255: '}';
while (i<LTSIZE) { if (ld<=0.0) ld = 0.0;
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1; //TRACE(" percent:%5.2f%% \n", ld*100.);
U32 end; //TRACE(" start:(%c)[%02X] ", character, character);
if (weight + i > LTSIZE) weight = LTSIZE-i; for (u=0; u<LTSIZE; ) {
end = i + weight; U32 const weight = (U32)((double)(LTSIZE - u) * ld) + 1;
while (i < end) lt[i++] = character; U32 const end = MIN ( u + weight , LTSIZE);
while (u < end) ldt[u++] = character; // TRACE(" %u(%c)[%02X] ", u, character, character);
character++; character++;
if (character > lastChar) character = firstChar; if (character > lastChar) character = firstChar;
} }
} }
static BYTE RDG_genChar(U32* seed, const litDistribTable lt) static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
{ {
U32 const id = RDG_rand(seed) & LTMASK; U32 const id = RDG_rand(seed) & LTMASK;
return (lt[id]); //TRACE(" %u : \n", id);
//TRACE(" %4u [%4u] ; val : %4u \n", id, id&255, ldt[id]);
return (ldt[id]); /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with 0.0. Checked : table is fully initialized */
} }
#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 0x7FFF) #define RDG_RAND15BITS ( RDG_rand(seed) & 0x7FFF )
#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15) #define RDG_RANDLENGTH ( (RDG_rand(seed) & 7) ? (RDG_rand(seed) & 0xF) : (RDG_rand(seed) & 0x1FF) + 0xF)
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr) void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
{ {
BYTE* buffPtr = (BYTE*)buffer; BYTE* buffPtr = (BYTE*)buffer;
const U32 matchProba32 = (U32)(32768 * matchProba); const U32 matchProba32 = (U32)(32768 * matchProba);
@@ -123,75 +129,73 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
} }
memset(buffPtr+pos, 0, size0); memset(buffPtr+pos, 0, size0);
pos += size0; pos += size0;
buffPtr[pos-1] = RDG_genChar(seed, lt); buffPtr[pos-1] = RDG_genChar(seed, ldt);
continue; continue;
} }
/* init */ /* init */
if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1; if (pos==0) buffPtr[0] = RDG_genChar(seed, ldt), pos=1;
/* Generate compressible data */ /* Generate compressible data */
while (pos < buffSize) { while (pos < buffSize) {
/* Select : Literal (char) or Match (within 32K) */ /* Select : Literal (char) or Match (within 32K) */
if (RDG_RAND15BITS < matchProba32) { if (RDG_RAND15BITS < matchProba32) {
/* Copy (within 32K) */ /* Copy (within 32K) */
size_t match; U32 const length = RDG_RANDLENGTH + 4;
size_t d; U32 const d = MIN (pos + length , buffSize);
size_t const length = RDG_RANDLENGTH + 4; U32 const repeatOffset = (RDG_rand(seed) & 15) == 2;
U32 offset = RDG_RAND15BITS + 1; U32 const randOffset = RDG_RAND15BITS + 1;
U32 repeatOffset = (RDG_rand(seed) & 15) == 2; U32 const offset = repeatOffset ? prevOffset : MIN(randOffset , pos);
if (repeatOffset) offset = prevOffset; size_t match = pos - offset;
if (offset > pos) offset = (U32)pos; //TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length);
prevOffset = offset;
match = pos - offset;
d = pos + length;
if (d > buffSize) d = buffSize;
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
prevOffset = offset;
} else { } else {
/* Literal (noise) */ /* Literal (noise) */
size_t const length = RDG_RANDLENGTH; U32 const length = RDG_RANDLENGTH;
size_t d = pos + length; U32 const d = MIN(pos + length, buffSize);
if (d > buffSize) d = buffSize; while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldt);
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt);
} } } }
} }
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
{ {
litDistribTable lt; BYTE ldt[LTSIZE];
if (litProba==0.0) litProba = matchProba / 4.5; memset(ldt, '0', sizeof(ldt));
RDG_fillLiteralDistrib(lt, litProba); if (litProba<=0.0) litProba = matchProba / 4.5;
RDG_genBlock(buffer, size, 0, matchProba, lt, &seed); //TRACE(" percent:%5.2f%% \n", litProba*100.);
RDG_fillLiteralDistrib(ldt, litProba);
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed);
} }
#define RDG_DICTSIZE (32 KB)
#define RDG_BLOCKSIZE (128 KB)
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed) void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
{ {
BYTE* buff = (BYTE*)malloc(RDG_DICTSIZE + RDG_BLOCKSIZE); size_t const stdBlockSize = 128 KB;
size_t const stdDictSize = 32 KB;
BYTE* buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
U64 total = 0; U64 total = 0;
litDistribTable ldt; BYTE ldt[LTSIZE];
/* init */ /* init */
if (buff==NULL) { fprintf(stdout, "not enough memory\n"); exit(1); } if (buff==NULL) { fprintf(stdout, "not enough memory\n"); exit(1); }
if (litProba<=0.0) litProba = matchProba / 4.5; if (litProba<=0.0) litProba = matchProba / 4.5;
memset(ldt, '0', sizeof(ldt));
RDG_fillLiteralDistrib(ldt, litProba); RDG_fillLiteralDistrib(ldt, litProba);
SET_BINARY_MODE(stdout); SET_BINARY_MODE(stdout);
/* Generate initial dict */ /* Generate initial dict */
RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldt, &seed); RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed);
/* Generate compressible data */ /* Generate compressible data */
while (total < size) { while (total < size) {
size_t const genBlockSize = (size_t) (MIN (RDG_BLOCKSIZE, size-total)); size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldt, &seed); RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed);
total += genBlockSize; total += genBlockSize;
{ size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; } { size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
/* update dict */ /* update dict */
memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE); memcpy(buff, buff + stdBlockSize, stdDictSize);
} }
/* cleanup */ /* cleanup */

22
programs/zbufftest.c
View File

@@ -140,7 +140,6 @@ static int basicUnitTests(U32 seed, double compressibility)
void* compressedBuffer = malloc(compressedBufferSize); void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize; size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize); void* decodedBuffer = malloc(decodedBufferSize);
U32 randState = seed;
size_t result, cSize, readSize, genSize; size_t result, cSize, readSize, genSize;
U32 testNb=0; U32 testNb=0;
ZBUFF_CCtx* zc = ZBUFF_createCCtx(); ZBUFF_CCtx* zc = ZBUFF_createCCtx();
@@ -151,7 +150,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAY("Not enough memory, aborting\n"); DISPLAY("Not enough memory, aborting\n");
goto _output_error; goto _output_error;
} }
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., randState); RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
/* Basic compression test */ /* Basic compression test */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
@@ -234,19 +233,13 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
{ {
const BYTE* b1 = (const BYTE*)buf1; const BYTE* b1 = (const BYTE*)buf1;
const BYTE* b2 = (const BYTE*)buf2; const BYTE* b2 = (const BYTE*)buf2;
size_t i; size_t u;
for (i=0; i<max; i++) { for (u=0; u<max; u++) {
if (b1[i] != b2[i]) break; if (b1[u] != b2[u]) break;
} }
return i; return u;
} }
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static size_t FUZ_rLogLength(U32* seed, U32 logLength) static size_t FUZ_rLogLength(U32* seed, U32 logLength)
{ {
size_t const lengthMask = ((size_t)1 << logLength) - 1; size_t const lengthMask = ((size_t)1 << logLength) - 1;
@@ -259,6 +252,11 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
return FUZ_rLogLength(seed, logLength); return FUZ_rLogLength(seed, logLength);
} }
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{ {
static const U32 maxSrcLog = 24; static const U32 maxSrcLog = 24;