lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

Updated FSE

Browse Source 
Added clang and g++ tests
This commit is contained in:
Yann Collet
2015-06-18 07:43:16 -08:00
parent bbfa7d77c9
commit 213089c078
11 changed files with 99 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -9,6 +9,8 @@ before_install:
env:
- ZSTD_TRAVIS_CI_ENV=travis-install
- ZSTD_TRAVIS_CI_ENV=test-all
- ZSTD_TRAVIS_CI_ENV=clangtest
- ZSTD_TRAVIS_CI_ENV=gpptest
matrix:
fast_finish: true

11
Makefile
View File

@ -32,7 +32,7 @@
# ################################################################
# Version number
export VERSION=0.0.1
export VERSION=0.0.2
export RELEASE=r$(VERSION)
DESTDIR?=
@ -51,6 +51,8 @@ TRAVIS_TARGET=$(ZSTD_TRAVIS_CI_ENV)
endif
.PHONY: clean
default: zstdprograms
all:
@ -90,4 +92,11 @@ test-travis: $(TRAVIS_TARGET)
prg-travis:
@cd $(PRGDIR); $(MAKE) -e $(ZSTD_TRAVIS_CI_ENV)
clangtest: clean
$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
gpptest: clean
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
endif

59
lib/fse.c
View File

@ -305,6 +305,8 @@ size_t FSE_headerBound(unsigned maxSymbolValue, unsigned tableLog)
return maxSymbolValue ? maxHeaderSize : FSE_MAX_HEADERSIZE;
}
#ifndef __clang_analyzer__ /* clang static analyzer has difficulties with this function : seems to believe normalizedCounter is uninitialized */
static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize,
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
unsigned safeWrite)
@ -341,7 +343,7 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize,
while (charnum >= start+24)
{
start+=24;
bitStream += 0xFFFF<<bitCount;
bitStream += 0xFFFFU << bitCount;
if ((!safeWrite) && (out > oend-2)) return (size_t)-FSE_ERROR_GENERIC; /* Buffer overflow */
out[0] = (BYTE) bitStream;
out[1] = (BYTE)(bitStream>>8);
@ -370,7 +372,7 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize,
short count = normalizedCounter[charnum++];
const short max = (short)((2*threshold-1)-remaining);
remaining -= FSE_abs(count);
if (remaining<0) return (size_t)-FSE_ERROR_GENERIC;
if (remaining<1) return (size_t)-FSE_ERROR_GENERIC;
count++; /* +1 for extra accuracy */
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
bitStream += count << bitCount;
@ -400,6 +402,7 @@ static size_t FSE_writeHeader_generic (void* header, size_t headerBufferSize,
return (out-ostart);
}
#endif // __clang_analyzer__
size_t FSE_writeHeader (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
@ -690,7 +693,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
U32 weight = sEnd - sStart;
if (weight < 1)
return (size_t)-FSE_ERROR_GENERIC;
norm[s] = weight;
norm[s] = (short)weight;
tmpTotal = end;
}
}
@ -852,7 +855,7 @@ void FSE_initCStream(FSE_CStream_t* bitC, void* start)
void FSE_initCState(FSE_CState_t* statePtr, const void* CTable)
{
const U32 tableLog = ( (U16*) CTable) [0];
const U32 tableLog = ( (const U16*) CTable) [0];
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = ((const U16*) CTable) + 2;
statePtr->symbolTT = (const U32*)CTable + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
@ -1038,7 +1041,7 @@ size_t FSE_decompressRLE(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize)
{
if (cSrcSize != 1) return (size_t)-FSE_ERROR_srcSize_wrong;
memset(dst, *(BYTE*)cSrc, originalSize);
memset(dst, *(const BYTE*)cSrc, originalSize);
return originalSize;
}
@ -1100,30 +1103,30 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz
if (srcSize >= sizeof(bitD_t))
{
U32 contain32;
bitD->start = (char*)srcBuffer;
bitD->ptr = (char*)srcBuffer + srcSize - sizeof(bitD_t);
bitD->start = (const char*)srcBuffer;
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD_t);
bitD->bitContainer = FSE_readLEST(bitD->ptr);
contain32 = ((BYTE*)srcBuffer)[srcSize-1];
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */
bitD->bitsConsumed = 8 - FSE_highbit32(contain32);
}
else
{
U32 contain32;
bitD->start = (char*)srcBuffer;
bitD->start = (const char*)srcBuffer;
bitD->ptr = bitD->start;
bitD->bitContainer = *(BYTE*)(bitD->start);
bitD->bitContainer = *(const BYTE*)(bitD->start);
switch(srcSize)
{
case 7: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[6]) << (sizeof(bitD_t)*8 - 16);
case 6: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[5]) << (sizeof(bitD_t)*8 - 24);
case 5: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[4]) << (sizeof(bitD_t)*8 - 32);
case 4: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[3]) << 24;
case 3: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[2]) << 16;
case 2: bitD->bitContainer += (bitD_t)(((BYTE*)(bitD->start))[1]) << 8;
case 7: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(bitD_t)*8 - 16);
case 6: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(bitD_t)*8 - 24);
case 5: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(bitD_t)*8 - 32);
case 4: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[3]) << 24;
case 3: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[2]) << 16;
case 2: bitD->bitContainer += (bitD_t)(((const BYTE*)(bitD->start))[1]) << 8;
default:;
}
contain32 = ((BYTE*)srcBuffer)[srcSize-1];
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */
bitD->bitsConsumed = 8 - FSE_highbit32(contain32);
bitD->bitsConsumed += (U32)(sizeof(bitD_t) - srcSize)*8;
@ -1135,18 +1138,19 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz
/* FSE_readBits
* Read next n bits from the bitContainer.
* Use the fast variant *only* if n > 0.
* Note : for this function to work properly on 32-bits, don't read more than maxNbBits==25
* On 32-bits, don't read more than maxNbBits==25
* On 64-bits, don't read more than maxNbBits==57
* Use the fast variant *only* if n >= 1.
* return : value extracted.
*/
bitD_t FSE_readBits(FSE_DStream_t* bitD, U32 nbBits)
{
bitD_t value = ((bitD->bitContainer << bitD->bitsConsumed) >> 1) >> (((sizeof(bitD_t)*8)-1)-nbBits);
bitD_t value = ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD_t)*8)-1))) >> 1) >> (((sizeof(bitD_t)*8)-1)-nbBits);
bitD->bitsConsumed += nbBits;
return value;
}
bitD_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 */
bitD_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */
{
bitD_t value = (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD_t)*8)-nbBits);
bitD->bitsConsumed += nbBits;
@ -1215,7 +1219,7 @@ BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD)
unsigned FSE_endOfDStream(const FSE_DStream_t* bitD)
{
return FSE_reloadDStream((FSE_DStream_t*)bitD)==2;
return ((bitD->ptr == bitD->start) && (bitD->bitsConsumed == sizeof(bitD_t)*8));
}
unsigned FSE_endOfDState(const FSE_DState_t* statePtr)
@ -1264,14 +1268,15 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
}
/* tail */
/* note : FSE_reloadDStream(&bitD) >= 1; Ends at exactly 2 */
while (1)
{
if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDState(&state1) && FSE_endOfDStream(&bitD)) )
if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state1))) )
break;
*op++ = fast ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDState(&state2) && FSE_endOfDStream(&bitD)) )
if ( (FSE_reloadDStream(&bitD)>2) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state2))) )
break;
*op++ = fast ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
@ -1442,6 +1447,8 @@ size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION) (unsigned* count, co
}
#ifndef __clang_analyzer__ /* clang static analyzer doesn't understand that tableSymbol is necessarily entirely initialized */
static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
@ -1533,6 +1540,8 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
return 0;
}
#endif // __clang_analyzer__
#define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION)
@ -1559,7 +1568,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
U32 position = 0;
U32 highThreshold = tableSize-1;
const S16 largeLimit= 1 << (tableLog-1);
const S16 largeLimit= (S16)(1 << (tableLog-1));
U32 noLarge = 1;
U32 s;

4
lib/zstd.h
View File

@ -70,7 +70,7 @@ ZSTD_compress() :
or an error code if it fails (which can be tested using ZSTD_isError())
ZSTD_decompress() :
compressedSize : is obviously the source size
compressedSize : is the exact source size
maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.
It must be equal or larger than originalSize, otherwise decompression will fail.
return : the number of bytes decompressed into destination buffer (originalSize)
@ -81,7 +81,7 @@ ZSTD_decompress() :
/**************************************
* Tool functions
**************************************/
size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size */
size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size (worst case scenario) */
/* Error Management */
unsigned ZSTD_isError(size_t code); /* tells if a return value is an error code */

2
programs/bench.c
View File

@ -471,7 +471,7 @@ static int BMK_syntheticTest(int cLevel, double compressibility)
}
int BMK_bench(char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
{
double compressibility = (double)g_compressibilityDefault / 100;

10
programs/bench.h
View File

@ -24,19 +24,11 @@
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/* Main function */
int BMK_bench(char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
/* Set Parameters */
void BMK_SetNbIterations(int nbLoops);
#if defined (__cplusplus)
}
#endif

15
programs/datagen.c
View File

@ -75,7 +75,7 @@
* Local Functions
*********************************************************/
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned int RDG_rand(U32* src)
static U32 RDG_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
@ -88,20 +88,21 @@ static unsigned int RDG_rand(U32* src)
#define LTSIZE 8192
#define LTMASK (LTSIZE-1)
static void* RDG_createLiteralDistrib(double ld)
static void* RDG_createLiteralDistrib(const double ld)
{
BYTE* lt = malloc(LTSIZE);
BYTE* lt = (BYTE*)malloc(LTSIZE);
U32 i = 0;
BYTE character = '0';
BYTE firstChar = '(';
BYTE lastChar = '}';
if (ld==0.0)
if (ld<=0.02)
{
character = 0;
firstChar = 0;
lastChar =255;
lastChar = 254;
character = firstChar;
}
if (ld==0.0) lastChar = 255;
while (i<LTSIZE)
{
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
@ -117,7 +118,7 @@ static void* RDG_createLiteralDistrib(double ld)
static char RDG_genChar(U32* seed, const void* ltctx)
{
const BYTE* lt = ltctx;
const BYTE* lt = (const BYTE*)ltctx;
U32 id = RDG_rand(seed) & LTMASK;
return lt[id];
}

14
programs/fileio.c
View File

@ -243,8 +243,8 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
ctx = ZSTD_createCCtx();
/* Allocate Memory */
inBuff = malloc(inBuffSize);
outBuff = malloc(outBuffSize);
inBuff = (BYTE*)malloc(inBuffSize);
outBuff = (BYTE*)malloc(outBuffSize);
if (!inBuff || !outBuff) EXM_THROW(21, "Allocation error : not enough memory");
inSlot = inBuff;
inEnd = inBuff + inBuffSize;
@ -265,6 +265,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* Fill input Buffer */
if (inSlot + blockSize > inEnd) inSlot = inBuff;
inSize = fread(inSlot, (size_t)1, blockSize, finput);
DISPLAY("Read block of size %u at pos %u \n", (U32)inSize, (U32)(inSlot-inBuff));
if (inSize==0) break;
filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
@ -340,9 +341,9 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
/* Allocate Memory */
inBuffSize = blockSize + FIO_blockHeaderSize;
inBuff = malloc(inBuffSize);
inBuff = (BYTE*)malloc(inBuffSize);
outBuffSize = wNbBlocks * blockSize;
outBuff = malloc(outBuffSize);
outBuff = (BYTE*)malloc(outBuffSize);
op = outBuff;
oend = outBuff + outBuffSize;
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
@ -352,6 +353,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
while (toRead)
{
size_t readSize, decodedSize;
static U32 nbReads = 0;
/* Fill input buffer */
readSize = fread(inBuff, 1, toRead, finput);
@ -359,11 +361,15 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
EXM_THROW(34, "Read error");
/* Decode block */
if (nbReads==55)
DISPLAY("!");
decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
DISPLAY("nbReads : %u \n", nbReads++);
if (decodedSize) /* not a header */
{
/* Write block */
DISPLAY("writing %u bytes from pos %u \n", (U32)decodedSize, (U32)(op-outBuff));
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file");
filesize += decodedSize;

6
programs/fullbench.c
View File

@ -267,7 +267,7 @@ size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void*
{
U32 i;
size_t total = 0;
BYTE* data = buff2;
BYTE* data = (BYTE*)buff2;
(void)dst; (void)dstSize; (void)src;
for (i=0; i < srcSize; i++)
@ -332,8 +332,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
/* Allocation */
dstBuffSize = ZSTD_compressBound(srcSize);
dstBuff = malloc(dstBuffSize);
buff2 = malloc(dstBuffSize);
dstBuff = (BYTE*)malloc(dstBuffSize);
buff2 = (BYTE*)malloc(dstBuffSize);
if ((!dstBuff) || (!buff2))
{
DISPLAY("\nError: not enough memory!\n");

29
programs/fuzzer.c
View File

@ -256,6 +256,25 @@ static int basicUnitTests(U32 seed, double compressibility)
if (result != (size_t)-ZSTD_ERROR_wrongMagicNumber) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* long rle test */
{
size_t sampleSize = 0;
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
FUZ_generateSynthetic(CNBuffer, sampleSize, compressibility, &randState);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1;
FUZ_generateSynthetic((char*)CNBuffer+sampleSize, 96 KB, compressibility, &randState);
sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize);
if (ZSTD_isError(cSize)) goto _output_error;
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
if (ZSTD_isError(result)) goto _output_error;
if (result!=sampleSize) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
}
_end:
free(CNBuffer);
free(compressedBuffer);
@ -271,8 +290,8 @@ _output_error:
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
{
const BYTE* b1 = buf1;
const BYTE* b2 = buf2;
const BYTE* b1 = (const BYTE*)buf1;
const BYTE* b2 = (const BYTE*)buf2;
size_t i;
for (i=0; i<max; i++)
{
@ -301,9 +320,9 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
(void)startTest; (void)compressibility;
/* allocation */
srcBuffer = malloc (srcBufferSize);
dstBuffer = malloc (dstBufferSize);
cBuffer = malloc (cBufferSize);
srcBuffer = (BYTE*)malloc (srcBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!srcBuffer || !dstBuffer || !cBuffer, "Not enough memory, fuzzer tests cancelled");
/* Create initial sample */

2
programs/zstdcli.c
View File

@ -271,7 +271,7 @@ int main(int argc, char** argv)
if (!strcmp(inFileName, stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
/* Check if benchmark is selected */
if (bench) { BMK_bench(argv+fileNameStart, nbFiles, 0); goto _end; }
if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, 0); goto _end; }
/* No output filename ==> try to select one automatically (when possible) */
while (!outFileName)