lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-11-03 20:33:11 +03:00
Code Activity

handles litLength >= 65535

Browse Source
This commit is contained in:
Yann Collet
2016-03-21 00:07:42 +01:00
parent 597847a2ae
commit d64f435f63
4 changed files with 19 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lib/bitstream.h
View File

@@ -206,7 +206,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBi
* unsafe version; does not check buffer overflow */ * unsafe version; does not check buffer overflow */
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
{ {
size_t nbBytes = bitC->bitPos >> 3; size_t const nbBytes = bitC->bitPos >> 3;
MEM_writeLEST(bitC->ptr, bitC->bitContainer); MEM_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes; bitC->ptr += nbBytes;
bitC->bitPos &= 7; bitC->bitPos &= 7;
@@ -218,7 +218,7 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
* note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
{ {
size_t nbBytes = bitC->bitPos >> 3; size_t const nbBytes = bitC->bitPos >> 3;
MEM_writeLEST(bitC->ptr, bitC->bitContainer); MEM_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes; bitC->ptr += nbBytes;
if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;

23
lib/zstd_compress.c
View File

@@ -172,9 +172,9 @@ size_t ZSTD_sizeofCCtx(ZSTD_parameters params) /* hidden interface, for parama
{ {
ZSTD_CCtx* zc = ZSTD_createCCtx(); ZSTD_CCtx* zc = ZSTD_createCCtx();
ZSTD_compressBegin_advanced(zc, NULL, 0, params); ZSTD_compressBegin_advanced(zc, NULL, 0, params);
{ size_t const size = sizeof(*zc) + zc->workSpaceSize; { size_t const ccsize = sizeof(*zc) + zc->workSpaceSize;
ZSTD_freeCCtx(zc); ZSTD_freeCCtx(zc);
return size; } return ccsize; }
} }
@@ -291,7 +291,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
/*! ZSTD_reduceTable() : /*! ZSTD_reduceTable() :
* rescale indexes from a table (indexes are U32) */ * reduce table indexes by `reducerValue` */
static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reducerValue) static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reducerValue)
{ {
U32 u; U32 u;
@@ -586,15 +586,15 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
FSE_CTable* CTable_MatchLength = zc->matchlengthCTable; FSE_CTable* CTable_MatchLength = zc->matchlengthCTable;
U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
const U16* const llTable = seqStorePtr->litLengthStart; const U16* const llTable = seqStorePtr->litLengthStart;
const U16* const llPtr = seqStorePtr->litLength;
const BYTE* const mlTable = seqStorePtr->matchLengthStart; const BYTE* const mlTable = seqStorePtr->matchLengthStart;
const U32* const offsetTable = seqStorePtr->offsetStart; const U32* const offsetTable = seqStorePtr->offsetStart;
const U32* const offsetTableEnd = seqStorePtr->offset;
BYTE* const offCodeTable = seqStorePtr->offCodeStart; BYTE* const offCodeTable = seqStorePtr->offCodeStart;
BYTE* const llCodeTable = seqStorePtr->llCodeStart; BYTE* const llCodeTable = seqStorePtr->llCodeStart;
BYTE* const ostart = (BYTE*)dst; BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + dstCapacity; BYTE* const oend = ostart + dstCapacity;
BYTE* op = ostart; BYTE* op = ostart;
size_t const nbSeq = llPtr - llTable; size_t const nbSeq = offsetTableEnd - offsetTable;
BYTE* seqHead; BYTE* seqHead;
/* Compress literals */ /* Compress literals */
@@ -648,7 +648,8 @@ static const BYTE deltaCode = 18;
{ size_t i; { size_t i;
for (i=0; i<nbSeq; i++) { for (i=0; i<nbSeq; i++) {
U32 const ll = llTable[i]; U32 ll = llTable[i];
if (llTable[i] == 65535) ll = seqStorePtr->litLengthLong;
llCodeTable[i] = (ll>63) ? ZSTD_highbit(ll) + deltaCode : llCode[ll]; llCodeTable[i] = (ll>63) ? ZSTD_highbit(ll) + deltaCode : llCode[ll];
} } } }
@@ -788,10 +789,11 @@ static const U32 llBits[36] = { 0, 0, 0, 0, 0, 0, 0, 0,
const BYTE LLCode = llCodeTable[n]; /* (7)*/ /* (7)*/ const BYTE LLCode = llCodeTable[n]; /* (7)*/ /* (7)*/
FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 17 */ /* 17 */ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 17 */ /* 17 */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */ if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
FSE_encodeSymbol(&blockStream, &stateLitLength, LLCode); /* 17 */ /* 27 */ FSE_encodeSymbol(&blockStream, &stateLitLength, LLCode); /* 16 */ /* 26 */
FSE_encodeSymbol(&blockStream, &stateOffsetBits, offCode); /* 26 */ /* 36 */ FSE_encodeSymbol(&blockStream, &stateOffsetBits, offCode); /* 25 */ /* 35 */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */ if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
BIT_addBits(&blockStream, offset, nbBits); /* 31 */ /* 62 */ /* 24 bits max in 32-bits mode */ //BIT_flushBits(&blockStream); /* 7 */ /* 7 */
BIT_addBits(&blockStream, offset, nbBits); /* 31 */ /* 61 */ /* 24 bits max in 32-bits mode */
BIT_addBits(&blockStream, llTable[n], llBits[LLCode]); BIT_addBits(&blockStream, llTable[n], llBits[LLCode]);
BIT_flushBits(&blockStream); /* 7 */ /* 7 */ BIT_flushBits(&blockStream); /* 7 */ /* 7 */
} } } }
@@ -866,7 +868,8 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
/* literal Length */ /* literal Length */
#if 1 #if 1
*seqStorePtr->litLength++ = (U16)litLength; /* take care of litLength >= 65535 ! */ if (litLength>=65535) { *(seqStorePtr->litLength++) = 65535; seqStorePtr->litLengthLong = (U32)litLength; }
else *seqStorePtr->litLength++ = (U16)litLength;
#else #else
if (litLength >= MaxLL) { if (litLength >= MaxLL) {
*(seqStorePtr->litLength++) = MaxLL; *(seqStorePtr->litLength++) = MaxLL;

3
lib/zstd_internal.h
View File

@@ -110,7 +110,7 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
#define MaxLL ((1<<LLbits) - 1) #define MaxLL ((1<<LLbits) - 1)
#define MaxOff ((1<<Offbits)- 1) #define MaxOff ((1<<Offbits)- 1)
#define MLFSELog 10 #define MLFSELog 10
#define LLFSELog 10 #define LLFSELog 9
#define OffFSELog 9 #define OffFSELog 9
#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ #define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
@@ -188,6 +188,7 @@ typedef struct {
BYTE* lit; BYTE* lit;
U16* litLengthStart; U16* litLengthStart;
U16* litLength; U16* litLength;
U32 litLengthLong;
BYTE* llCodeStart; BYTE* llCodeStart;
BYTE* matchLengthStart; BYTE* matchLengthStart;
BYTE* matchLength; BYTE* matchLength;

4
programs/bench.c
View File

@@ -188,8 +188,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, U32 nbFiles, const size_t* fileSizes, U32 nbFiles,
const void* dictBuffer, size_t dictBufferSize) const void* dictBuffer, size_t dictBufferSize)
{ {
const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize); void* const compressedBuffer = malloc(maxCompressedSize);