diff --git a/.travis.yml b/.travis.yml index f302fabe4..17846f160 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,8 @@ before_install: env: - ZSTD_TRAVIS_CI_ENV=travis-install - ZSTD_TRAVIS_CI_ENV=test-all + - ZSTD_TRAVIS_CI_ENV=clangtest + - ZSTD_TRAVIS_CI_ENV=gpptest matrix: fast_finish: true diff --git a/Makefile b/Makefile index ecb5f17bc..adc6d5e8a 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ # ################################################################ # Version number -export VERSION=0.0.1 +export VERSION=0.0.2 export RELEASE=r$(VERSION) DESTDIR?= @@ -51,6 +51,8 @@ TRAVIS_TARGET=$(ZSTD_TRAVIS_CI_ENV) endif +.PHONY: clean + default: zstdprograms all: @@ -90,4 +92,11 @@ test-travis: $(TRAVIS_TARGET) prg-travis: @cd $(PRGDIR); $(MAKE) -e $(ZSTD_TRAVIS_CI_ENV) +clangtest: clean + $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" + +gpptest: clean + $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + + endif diff --git a/lib/fse.c b/lib/fse.c index 2861fd9f1..a81e2f381 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -305,6 +305,8 @@ size_t FSE_headerBound(unsigned maxSymbolValue, unsigned tableLog) return maxSymbolValue ? maxHeaderSize : FSE_MAX_HEADERSIZE; } +#ifndef __clang_analyzer__ /* clang static analyzer has difficulties with this function : seems to believe normalizedCounter is uninitialized */ + static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, unsigned safeWrite) @@ -341,9 +343,9 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, while (charnum >= start+24) { start+=24; - bitStream += 0xFFFF< oend-2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */ - out[0] = (BYTE)bitStream; + out[0] = (BYTE) bitStream; out[1] = (BYTE)(bitStream>>8); out+=2; bitStream>>=16; @@ -370,7 +372,7 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, short count = normalizedCounter[charnum++]; const short max = (short)((2*threshold-1)-remaining); remaining -= FSE_abs(count); - if (remaining<0) return (size_t)-FSE_ERROR_GENERIC; + if (remaining<1) return (size_t)-FSE_ERROR_GENERIC; count++; /* +1 for extra accuracy */ if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ bitStream += count << bitCount; @@ -400,6 +402,7 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize, return (out-ostart); } +#endif // __clang_analyzer__ size_t FSE_writeHeader (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) @@ -690,7 +693,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, U32 weight = sEnd - sStart; if (weight < 1) return (size_t)-FSE_ERROR_GENERIC; - norm[s] = weight; + norm[s] = (short)weight; tmpTotal = end; } } @@ -852,7 +855,7 @@ void FSE_initCStream(FSE_CStream_t* bitC, void* start) void FSE_initCState(FSE_CState_t* statePtr, const void* CTable) { - const U32 tableLog = ( (U16*) CTable) [0]; + const U32 tableLog = ( (const U16*) CTable) [0]; statePtr->value = (ptrdiff_t)1<stateTable = ((const U16*) CTable) + 2; statePtr->symbolTT = (const U32*)CTable + 1 + (tableLog ? (1<<(tableLog-1)) : 1); @@ -1038,7 +1041,7 @@ size_t FSE_decompressRLE(void* dst, size_t originalSize, const void* cSrc, size_t cSrcSize) { if (cSrcSize != 1) return (size_t)-FSE_ERROR_srcSize_wrong; - memset(dst, *(BYTE*)cSrc, originalSize); + memset(dst, *(const BYTE*)cSrc, originalSize); return originalSize; } @@ -1100,30 +1103,30 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz if (srcSize >= sizeof(bitD_t)) { U32 contain32; - bitD->start = (char*)srcBuffer; - bitD->ptr = (char*)srcBuffer + srcSize - sizeof(bitD_t); + bitD->start = (const char*)srcBuffer; + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD_t); bitD->bitContainer = FSE_readLEST(bitD->ptr); - contain32 = ((BYTE*)srcBuffer)[srcSize-1]; + contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */ bitD->bitsConsumed = 8 - FSE_highbit32(contain32); } else { U32 contain32; - bitD->start = (char*)srcBuffer; + bitD->start = (const char*)srcBuffer; bitD->ptr = bitD->start; - bitD->bitContainer = *(BYTE*)(bitD->start); + bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[6]) << (sizeof(bitD_t)*8 - 16); - case 6: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[5]) << (sizeof(bitD_t)*8 - 24); - case 5: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[4]) << (sizeof(bitD_t)*8 - 32); - case 4: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[3]) << 24; - case 3: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[2]) << 16; - case 2: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[1]) << 8; + case 7: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(bitD_t)*8 - 16); + case 6: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(bitD_t)*8 - 24); + case 5: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(bitD_t)*8 - 32); + case 4: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[3]) << 24; + case 3: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[2]) << 16; + case 2: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[1]) << 8; default:; } - contain32 = ((BYTE*)srcBuffer)[srcSize-1]; + contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */ bitD->bitsConsumed = 8 - FSE_highbit32(contain32); bitD->bitsConsumed += (U32)(sizeof(bitD_t) - srcSize)*8; @@ -1135,18 +1138,19 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz /* FSE_readBits * Read next n bits from the bitContainer. - * Use the fast variant *only* if n > 0. - * Note : for this function to work properly on 32-bits, don't read more than maxNbBits==25 + * On 32-bits, don't read more than maxNbBits==25 + * On 64-bits, don't read more than maxNbBits==57 + * Use the fast variant *only* if n >= 1. * return : value extracted. */ bitD_t FSE_readBits(FSE_DStream_t* bitD, U32 nbBits) { - bitD_t value = ((bitD->bitContainer << bitD->bitsConsumed) >> 1) >> (((sizeof(bitD_t)*8)-1)-nbBits); + bitD_t value = ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD_t)*8)-1))) >> 1) >> (((sizeof(bitD_t)*8)-1)-nbBits); bitD->bitsConsumed += nbBits; return value; } -bitD_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 */ +bitD_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ { bitD_t value = (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD_t)*8)-nbBits); bitD->bitsConsumed += nbBits; @@ -1215,7 +1219,7 @@ BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD) unsigned FSE_endOfDStream(const FSE_DStream_t* bitD) { - return FSE_reloadDStream((FSE_DStream_t*)bitD)==2; + return ((bitD->ptr == bitD->start) && (bitD->bitsConsumed == sizeof(bitD_t)*8)); } unsigned FSE_endOfDState(const FSE_DState_t* statePtr) @@ -1264,14 +1268,15 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic( } /* tail */ + /* note : FSE_reloadDStream(&bitD) >= 1; Ends at exactly 2 */ while (1) { - if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDState(&state1) && FSE_endOfDStream(&bitD)) ) + if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state1))) ) break; *op++ = fast ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD); - if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDState(&state2) && FSE_endOfDStream(&bitD)) ) + if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state2))) ) break; *op++ = fast ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD); @@ -1442,6 +1447,8 @@ size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION) (unsigned* count, co } +#ifndef __clang_analyzer__ /* clang static analyzer doesn't understand that tableSymbol is necessarily entirely initialized */ + static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) @@ -1533,6 +1540,8 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) return 0; } +#endif // __clang_analyzer__ + #define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION) @@ -1559,7 +1568,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; U32 position = 0; U32 highThreshold = tableSize-1; - const S16 largeLimit= 1 << (tableLog-1); + const S16 largeLimit= (S16)(1 << (tableLog-1)); U32 noLarge = 1; U32 s; diff --git a/lib/zstd.h b/lib/zstd.h index cda16b88a..50ee72c43 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -48,7 +48,7 @@ extern "C" { #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */ #define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ -#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) unsigned ZSTD_versionNumber (void); @@ -70,7 +70,7 @@ ZSTD_compress() : or an error code if it fails (which can be tested using ZSTD_isError()) ZSTD_decompress() : - compressedSize : is obviously the source size + compressedSize : is the exact source size maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. It must be equal or larger than originalSize, otherwise decompression will fail. return : the number of bytes decompressed into destination buffer (originalSize) @@ -81,7 +81,7 @@ ZSTD_decompress() : /************************************** * Tool functions **************************************/ -size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size */ +size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size (worst case scenario) */ /* Error Management */ unsigned ZSTD_isError(size_t code); /* tells if a return value is an error code */ diff --git a/programs/bench.c b/programs/bench.c index 85f4993db..31a43f320 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -72,7 +72,7 @@ /************************************** -* Basic Types +* Basic Types **************************************/ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include @@ -471,7 +471,7 @@ static int BMK_syntheticTest(int cLevel, double compressibility) } -int BMK_bench(char** fileNamesTable, unsigned nbFiles, unsigned cLevel) +int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel) { double compressibility = (double)g_compressibilityDefault / 100; diff --git a/programs/bench.h b/programs/bench.h index e07f37ff9..664c034e2 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -24,19 +24,11 @@ */ #pragma once -#if defined (__cplusplus) -extern "C" { -#endif - /* Main function */ -int BMK_bench(char** fileNamesTable, unsigned nbFiles, unsigned cLevel); +int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel); /* Set Parameters */ void BMK_SetNbIterations(int nbLoops); - -#if defined (__cplusplus) -} -#endif diff --git a/programs/datagen.c b/programs/datagen.c index 2a10b8113..20eb30e99 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -75,7 +75,7 @@ * Local Functions *********************************************************/ #define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r))) -static unsigned int RDG_rand(U32* src) +static U32 RDG_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; @@ -88,20 +88,21 @@ static unsigned int RDG_rand(U32* src) #define LTSIZE 8192 #define LTMASK (LTSIZE-1) -static void* RDG_createLiteralDistrib(double ld) +static void* RDG_createLiteralDistrib(const double ld) { - BYTE* lt = malloc(LTSIZE); + BYTE* lt = (BYTE*)malloc(LTSIZE); U32 i = 0; BYTE character = '0'; BYTE firstChar = '('; BYTE lastChar = '}'; - if (ld==0.0) + if (ld<=0.02) { - character = 0; firstChar = 0; - lastChar =255; + lastChar = 254; + character = firstChar; } + if (ld==0.0) lastChar = 255; while (i inEnd) inSlot = inBuff; inSize = fread(inSlot, (size_t)1, blockSize, finput); + DISPLAY("Read block of size %u at pos %u \n", (U32)inSize, (U32)(inSlot-inBuff)); if (inSize==0) break; filesize += inSize; DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20)); @@ -340,9 +341,9 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha /* Allocate Memory */ inBuffSize = blockSize + FIO_blockHeaderSize; - inBuff = malloc(inBuffSize); + inBuff = (BYTE*)malloc(inBuffSize); outBuffSize = wNbBlocks * blockSize; - outBuff = malloc(outBuffSize); + outBuff = (BYTE*)malloc(outBuffSize); op = outBuff; oend = outBuff + outBuffSize; if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory"); @@ -352,6 +353,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha while (toRead) { size_t readSize, decodedSize; + static U32 nbReads = 0; /* Fill input buffer */ readSize = fread(inBuff, 1, toRead, finput); @@ -359,11 +361,15 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha EXM_THROW(34, "Read error"); /* Decode block */ + if (nbReads==55) + DISPLAY("!"); decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize); + DISPLAY("nbReads : %u \n", nbReads++); if (decodedSize) /* not a header */ { /* Write block */ + DISPLAY("writing %u bytes from pos %u \n", (U32)decodedSize, (U32)(op-outBuff)); sizeCheck = fwrite(op, 1, decodedSize, foutput); if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file"); filesize += decodedSize; diff --git a/programs/fullbench.c b/programs/fullbench.c index b94918950..7afd12c88 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -267,7 +267,7 @@ size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* { U32 i; size_t total = 0; - BYTE* data = buff2; + BYTE* data = (BYTE*)buff2; (void)dst; (void)dstSize; (void)src; for (i=0; i < srcSize; i++) @@ -332,8 +332,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb) /* Allocation */ dstBuffSize = ZSTD_compressBound(srcSize); - dstBuff = malloc(dstBuffSize); - buff2 = malloc(dstBuffSize); + dstBuff = (BYTE*)malloc(dstBuffSize); + buff2 = (BYTE*)malloc(dstBuffSize); if ((!dstBuff) || (!buff2)) { DISPLAY("\nError: not enough memory!\n"); diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 91218b9e7..b2dbcd3d9 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -256,6 +256,25 @@ static int basicUnitTests(U32 seed, double compressibility) if (result != (size_t)-ZSTD_ERROR_wrongMagicNumber) goto _output_error; DISPLAYLEVEL(4, "OK \n"); + /* long rle test */ + { + size_t sampleSize = 0; + DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++); + FUZ_generateSynthetic(CNBuffer, sampleSize, compressibility, &randState); + memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); + sampleSize += 256 KB - 1; + FUZ_generateSynthetic((char*)CNBuffer+sampleSize, 96 KB, compressibility, &randState); + sampleSize += 96 KB; + cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize); + if (ZSTD_isError(cSize)) goto _output_error; + result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize); + if (ZSTD_isError(result)) goto _output_error; + if (result!=sampleSize) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + } + + + _end: free(CNBuffer); free(compressedBuffer); @@ -271,8 +290,8 @@ _output_error: static size_t findDiff(const void* buf1, const void* buf2, size_t max) { - const BYTE* b1 = buf1; - const BYTE* b2 = buf2; + const BYTE* b1 = (const BYTE*)buf1; + const BYTE* b2 = (const BYTE*)buf2; size_t i; for (i=0; i try to select one automatically (when possible) */ while (!outFileName)