lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

Added Wcast-qual compilation flag

Browse Source 
Updated xxHash
Removed log traces
This commit is contained in:
Yann Collet
2015-06-20 19:37:53 -08:00
parent fc774d31d1
commit 078a9a2804
3 changed files with 23 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
lib/zstd.c
View File

@ -178,11 +178,11 @@ static unsigned ZSTD_isLittleEndian(void)
return one.c[0]; return one.c[0];
} }
static U16 ZSTD_read16(const void* p) { return *(U16*)p; } static U16 ZSTD_read16(const void* p) { return *(const U16*)p; }
static U32 ZSTD_read32(const void* p) { return *(U32*)p; } static U32 ZSTD_read32(const void* p) { return *(const U32*)p; }
static size_t ZSTD_read_ARCH(const void* p) { return *(size_t*)p; } static size_t ZSTD_read_ARCH(const void* p) { return *(const size_t*)p; }
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
@ -503,7 +503,7 @@ static size_t ZSTD_compressRle (void* dst, size_t maxDstSize, const void* src, s
/* at this stage : dstSize >= FSE_compressBound(srcSize) > (ZSTD_blockHeaderSize+1) (checked by ZSTD_compressLiterals()) */ /* at this stage : dstSize >= FSE_compressBound(srcSize) > (ZSTD_blockHeaderSize+1) (checked by ZSTD_compressLiterals()) */
(void)maxDstSize; (void)maxDstSize;
ostart[ZSTD_blockHeaderSize] = *(BYTE*)src; ostart[ZSTD_blockHeaderSize] = *(const BYTE*)src;
/* Build header */ /* Build header */
ostart[0] = (BYTE)(srcSize>>16); ostart[0] = (BYTE)(srcSize>>16);
@ -621,9 +621,7 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
errorCode = FSE_count (count, ip, srcSize, &maxSymbolValue); errorCode = FSE_count (count, ip, srcSize, &maxSymbolValue);
if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC; if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
if (errorCode == srcSize) return 1; if (errorCode == srcSize) return 1;
//if (errorCode < ((srcSize * 7) >> 10)) return 0; if (errorCode < (srcSize >> 6)) return 0; /* cheap heuristic : probably not compressible enough */
//if (errorCode < (srcSize >> 7)) return 0;
if (errorCode < (srcSize >> 6)) return 0; /* heuristic : probably not compressible enough */
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
errorCode = (int)FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue); errorCode = (int)FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue);
@ -915,7 +913,7 @@ static const U64 prime7bytes = 58295818150454627ULL;
//static U32 ZSTD_hashPtr(const void* p) { return ( ((*(U64*)p & 0xFFFFFFFFFFFFFF) * prime7bytes) >> (64-HASH_LOG)); } //static U32 ZSTD_hashPtr(const void* p) { return ( ((*(U64*)p & 0xFFFFFFFFFFFFFF) * prime7bytes) >> (64-HASH_LOG)); }
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime8bytes) >> (64-HASH_LOG)); } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime8bytes) >> (64-HASH_LOG)); }
static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; } static U32 ZSTD_hashPtr(const void* p) { return ( (*(const U64*)p * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; }
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime6bytes) >> (48-HASH_LOG)) & HASH_MASK; } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime6bytes) >> (48-HASH_LOG)) & HASH_MASK; }
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime5bytes) >> (40-HASH_LOG)) & HASH_MASK; } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime5bytes) >> (40-HASH_LOG)) & HASH_MASK; }
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U32*)p * KNUTH) >> (32-HASH_LOG)); } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U32*)p * KNUTH) >> (32-HASH_LOG)); }
@ -961,7 +959,7 @@ static size_t ZSTD_compressBlock(void* cctx, void* dst, size_t maxDstSize, const
/* Main Search Loop */ /* Main Search Loop */
while (ip < ilimit) while (ip < ilimit)
{ {
const BYTE* match = (BYTE*) ZSTD_updateMatch(HashTable, ip, base); const BYTE* match = (const BYTE*) ZSTD_updateMatch(HashTable, ip, base);
if (!ZSTD_checkMatch(match,ip)) { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; } if (!ZSTD_checkMatch(match,ip)) { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; }
@ -1367,9 +1365,9 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx,
} }
case bt_compressed: case bt_compressed:
{ {
size_t cSize = ZSTD_decompressLiterals(ctx, dst, maxDstSize, ip, litcSize); size_t litSize = ZSTD_decompressLiterals(ctx, dst, maxDstSize, ip, litcSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(litSize)) return litSize;
*litPtr = oend - cSize; *litPtr = oend - litSize;
ip += litcSize; ip += litcSize;
break; break;
} }
@ -1508,7 +1506,7 @@ FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize
ip, iend-ip); ip, iend-ip);
if (ZSTD_isError(errorCode)) return errorCode; if (ZSTD_isError(errorCode)) return errorCode;
/* end pos */ /* end pos */
if ((litPtr>=ostart) && (litPtr<=oend)) if ((litPtr>=ostart) && (litPtr<=oend)) /* decoded literals are into dst buffer */
litEnd = oend - lastLLSize; litEnd = oend - lastLLSize;
else else
litEnd = ip - lastLLSize; litEnd = ip - lastLLSize;
@ -1519,7 +1517,6 @@ FORCE_INLINE size_t ZSTD_decompressBlock(void* ctx, void* dst, size_t maxDstSize
FSE_DStream_t DStream; FSE_DStream_t DStream;
FSE_DState_t stateLL, stateOffb, stateML; FSE_DState_t stateLL, stateOffb, stateML;
size_t prevOffset = 0, offset = 0; size_t prevOffset = 0, offset = 0;
size_t qutt=0;
FSE_initDStream(&DStream, ip, iend-ip); FSE_initDStream(&DStream, ip, iend-ip);
FSE_initDState(&stateLL, &DStream, DTableLL); FSE_initDState(&stateLL, &DStream, DTableLL);
@ -1545,7 +1542,6 @@ _another_round:
if (add < 255) litLength += add; if (add < 255) litLength += add;
else else
{ {
//litLength = (*(U32*)dumps) & 0xFFFFFF;
litLength = ZSTD_readLE32(dumps) & 0xFFFFFF; litLength = ZSTD_readLE32(dumps) & 0xFFFFFF;
dumps += 3; dumps += 3;
} }
@ -1578,7 +1574,7 @@ _another_round:
if (add < 255) matchLength += add; if (add < 255) matchLength += add;
else else
{ {
matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
dumps += 3; dumps += 3;
} }
} }
@ -1587,8 +1583,10 @@ _another_round:
/* copy Match */ /* copy Match */
{ {
BYTE* const endMatch = op + matchLength; BYTE* const endMatch = op + matchLength;
size_t qutt=0;
U64 saved[2]; U64 saved[2];
/* save beginning of literal sequence, in case of write overlap */
if ((size_t)(litPtr - endMatch) < 12) if ((size_t)(litPtr - endMatch) < 12)
{ {
qutt = endMatch + 12 - litPtr; qutt = endMatch + 12 - litPtr;
@ -1624,7 +1622,7 @@ _another_round:
op = endMatch; op = endMatch;
if ((size_t)(litPtr - endMatch) < 12) if ((size_t)(litPtr - endMatch) < 12)
memcpy((void*)litPtr, saved, qutt); memcpy(endMatch + (litPtr - endMatch), saved, qutt); /* required as litPtr is const ptr */
} }
} }

15
programs/Makefile
View File

@ -30,12 +30,12 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ########################################################################## # ##########################################################################
RELEASE?= r1 RELEASE?= v0.0.2
DESTDIR?= DESTDIR?=
PREFIX ?= /usr PREFIX ?= /usr
CFLAGS ?= -O3 CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\" CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\"
FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
BINDIR=$(PREFIX)/bin BINDIR=$(PREFIX)/bin
@ -142,13 +142,14 @@ test-fuzzer32: fuzzer32
./fuzzer32 ./fuzzer32
test-mem: zstd datagen fuzzer fullbench test-mem: zstd datagen fuzzer fullbench
valgrind --leak-check=yes ./datagen -g50M > /dev/null @echo "\n ---- valgrind tests : memory analyzer ----"
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > /dev/null
./datagen -g16KB > tmp ./datagen -g16KB > tmp
valgrind --leak-check=yes ./zstd -vf tmp /dev/null valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null
./datagen -g128MB > tmp ./datagen -g128MB > tmp
valgrind --leak-check=yes ./zstd -vf tmp /dev/null valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null
@rm tmp @rm tmp
valgrind --leak-check=yes ./fuzzer -i128 -t1 valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i128 -t1
valgrind --leak-check=yes ./fullbench -i1 valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1
endif endif

6
programs/fileio.c
View File

@ -265,7 +265,6 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* Fill input Buffer */ /* Fill input Buffer */
if (inSlot + blockSize > inEnd) inSlot = inBuff; if (inSlot + blockSize > inEnd) inSlot = inBuff;
inSize = fread(inSlot, (size_t)1, blockSize, finput); inSize = fread(inSlot, (size_t)1, blockSize, finput);
DISPLAY("Read block of size %u at pos %u \n", (U32)inSize, (U32)(inSlot-inBuff));
if (inSize==0) break; if (inSize==0) break;
filesize += inSize; filesize += inSize;
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20)); DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
@ -353,7 +352,6 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
while (toRead) while (toRead)
{ {
size_t readSize, decodedSize; size_t readSize, decodedSize;
static U32 nbReads = 0;
/* Fill input buffer */ /* Fill input buffer */
readSize = fread(inBuff, 1, toRead, finput); readSize = fread(inBuff, 1, toRead, finput);
@ -361,15 +359,11 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
EXM_THROW(34, "Read error"); EXM_THROW(34, "Read error");
/* Decode block */ /* Decode block */
if (nbReads==55)
DISPLAY("!");
decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize); decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
DISPLAY("nbReads : %u \n", nbReads++);
if (decodedSize) /* not a header */ if (decodedSize) /* not a header */
{ {
/* Write block */ /* Write block */
DISPLAY("writing %u bytes from pos %u \n", (U32)decodedSize, (U32)(op-outBuff));
sizeCheck = fwrite(op, 1, decodedSize, foutput); sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file"); if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file");
filesize += decodedSize; filesize += decodedSize;