lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

updated fse version

Browse Source 
feature minor refactoring (removing FSE_abs())
also : fix a few minor issues recently introduced in examples
This commit is contained in:
Yann Collet
2017-02-15 12:00:03 -08:00
parent c09d16ba8c
commit 4596037042
6 changed files with 80 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
examples/dictionary_decompression.c
View File

@ -13,6 +13,7 @@
#include <string.h> // strerror #include <string.h> // strerror
#include <errno.h> // errno #include <errno.h> // errno
#include <sys/stat.h> // stat #include <sys/stat.h> // stat
#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed

6
examples/simple_compression.c
View File

@ -102,7 +102,7 @@ static void compress_orDie(const char* fname, const char* oname)
} }
static const char* createOutFilename_orDie(const char* filename) static char* createOutFilename_orDie(const char* filename)
{ {
size_t const inL = strlen(filename); size_t const inL = strlen(filename);
size_t const outL = inL + 5; size_t const outL = inL + 5;
@ -110,7 +110,7 @@ static const char* createOutFilename_orDie(const char* filename)
memset(outSpace, 0, outL); memset(outSpace, 0, outL);
strcat(outSpace, filename); strcat(outSpace, filename);
strcat(outSpace, ".zst"); strcat(outSpace, ".zst");
return (const char*)outSpace; return (char*)outSpace;
} }
int main(int argc, const char** argv) int main(int argc, const char** argv)
@ -125,7 +125,7 @@ int main(int argc, const char** argv)
return 1; return 1;
} }
const char* const outFilename = createOutFilename_orDie(inFilename); char* const outFilename = createOutFilename_orDie(inFilename);
compress_orDie(inFilename, outFilename); compress_orDie(inFilename, outFilename);
free(outFilename); free(outFilename);
return 0; return 0;

21
examples/simple_decompression.c
View File

@ -6,17 +6,16 @@
* LICENSE-examples file in the root directory of this source tree. * LICENSE-examples file in the root directory of this source tree.
*/ */
#include <stdlib.h> // malloc, exit #include <stdlib.h> // malloc, exit
#include <stdio.h> // printf #include <stdio.h> // printf
#include <string.h> // strerror #include <string.h> // strerror
#include <errno.h> // errno #include <errno.h> // errno
#include <sys/stat.h> // stat #include <sys/stat.h> // stat
#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
static off_t fsize_X(const char *filename) static off_t fsize_orDie(const char *filename)
{ {
struct stat st; struct stat st;
if (stat(filename, &st) == 0) return st.st_size; if (stat(filename, &st) == 0) return st.st_size;
@ -25,7 +24,7 @@ static off_t fsize_X(const char *filename)
exit(1); exit(1);
} }
static FILE* fopen_X(const char *filename, const char *instruction) static FILE* fopen_orDie(const char *filename, const char *instruction)
{ {
FILE* const inFile = fopen(filename, instruction); FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile; if (inFile) return inFile;
@ -34,7 +33,7 @@ static FILE* fopen_X(const char *filename, const char *instruction)
exit(2); exit(2);
} }
static void* malloc_X(size_t size) static void* malloc_orDie(size_t size)
{ {
void* const buff = malloc(size); void* const buff = malloc(size);
if (buff) return buff; if (buff) return buff;
@ -43,11 +42,11 @@ static void* malloc_X(size_t size)
exit(3); exit(3);
} }
static void* loadFile_X(const char* fileName, size_t* size) static void* loadFile_orDie(const char* fileName, size_t* size)
{ {
off_t const buffSize = fsize_X(fileName); off_t const buffSize = fsize_orDie(fileName);
FILE* const inFile = fopen_X(fileName, "rb"); FILE* const inFile = fopen_orDie(fileName, "rb");
void* const buffer = malloc_X(buffSize); void* const buffer = malloc_orDie(buffSize);
size_t const readSize = fread(buffer, 1, buffSize, inFile); size_t const readSize = fread(buffer, 1, buffSize, inFile);
if (readSize != (size_t)buffSize) { if (readSize != (size_t)buffSize) {
printf("fread: %s : %s \n", fileName, strerror(errno)); printf("fread: %s : %s \n", fileName, strerror(errno));
@ -62,13 +61,13 @@ static void* loadFile_X(const char* fileName, size_t* size)
static void decompress(const char* fname) static void decompress(const char* fname)
{ {
size_t cSize; size_t cSize;
void* const cBuff = loadFile_X(fname, &cSize); void* const cBuff = loadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==0) { if (rSize==0) {
printf("%s : original size unknown. Use streaming decompression instead. \n", fname); printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
exit(5); exit(5);
} }
void* const rBuff = malloc_X((size_t)rSize); void* const rBuff = malloc_orDie((size_t)rSize);
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize); size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);

20
lib/common/entropy_common.c
View File

@ -43,6 +43,12 @@
#include "huf.h" #include "huf.h"
/*-****************************************
* Version
******************************************/
unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
/*-**************************************** /*-****************************************
* FSE Error Management * FSE Error Management
******************************************/ ******************************************/
@ -62,8 +68,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*-************************************************************** /*-**************************************************************
* FSE NCount encoding-decoding * FSE NCount encoding-decoding
****************************************************************/ ****************************************************************/
static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
const void* headerBuffer, size_t hbSize) const void* headerBuffer, size_t hbSize)
{ {
@ -117,21 +121,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
} else { } else {
bitStream >>= 2; bitStream >>= 2;
} } } }
{ short const max = (short)((2*threshold-1)-remaining); { int const max = (2*threshold-1) - remaining;
short count; int count;
if ((bitStream & (threshold-1)) < (U32)max) { if ((bitStream & (threshold-1)) < (U32)max) {
count = (short)(bitStream & (threshold-1)); count = bitStream & (threshold-1);
bitCount += nbBits-1; bitCount += nbBits-1;
} else { } else {
count = (short)(bitStream & (2*threshold-1)); count = bitStream & (2*threshold-1);
if (count >= threshold) count -= max; if (count >= threshold) count -= max;
bitCount += nbBits; bitCount += nbBits;
} }
count--; /* extra accuracy */ count--; /* extra accuracy */
remaining -= FSE_abs(count); remaining -= count < 0 ? -count : count; /* -1 means +1 */
normalizedCounter[charnum++] = count; normalizedCounter[charnum++] = (short)count;
previous0 = !count; previous0 = !count;
while (remaining < threshold) { while (remaining < threshold) {
nbBits--; nbBits--;

66
lib/common/fse.h
View File

@ -45,6 +45,32 @@ extern "C" {
#include <stddef.h> /* size_t, ptrdiff_t */ #include <stddef.h> /* size_t, ptrdiff_t */
/*-*****************************************
* FSE_PUBLIC_API : control library symbols visibility
******************************************/
#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
# define FSE_PUBLIC_API __declspec(dllexport)
#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define FSE_PUBLIC_API
#endif
/*------ Version ------*/
#define FSE_VERSION_MAJOR 0
#define FSE_VERSION_MINOR 9
#define FSE_VERSION_RELEASE 0
#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
#define FSE_QUOTE(str) #str
#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
/*-**************************************** /*-****************************************
* FSE simple functions * FSE simple functions
******************************************/ ******************************************/
@ -56,7 +82,7 @@ extern "C" {
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
if FSE_isError(return), compression failed (more details using FSE_getErrorName()) if FSE_isError(return), compression failed (more details using FSE_getErrorName())
*/ */
size_t FSE_compress(void* dst, size_t dstCapacity, FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize); const void* src, size_t srcSize);
/*! FSE_decompress(): /*! FSE_decompress():
@ -69,18 +95,18 @@ size_t FSE_compress(void* dst, size_t dstCapacity,
Why ? : making this distinction requires a header. Why ? : making this distinction requires a header.
Header management is intentionally delegated to the user layer, which can better manage special cases. Header management is intentionally delegated to the user layer, which can better manage special cases.
*/ */
size_t FSE_decompress(void* dst, size_t dstCapacity, FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
const void* cSrc, size_t cSrcSize); const void* cSrc, size_t cSrcSize);
/*-***************************************** /*-*****************************************
* Tool functions * Tool functions
******************************************/ ******************************************/
size_t FSE_compressBound(size_t size); /* maximum compressed size */ FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
/* Error Management */ /* Error Management */
unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/*-***************************************** /*-*****************************************
@ -94,7 +120,7 @@ const char* FSE_getErrorName(size_t code); /* provides error code string (usef
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
if FSE_isError(return), it's an error code. if FSE_isError(return), it's an error code.
*/ */
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
/*-***************************************** /*-*****************************************
@ -127,50 +153,50 @@ or to save and provide normalized distribution using external method.
@return : the count of the most frequent symbol (which is not identified). @return : the count of the most frequent symbol (which is not identified).
if return == srcSize, there is only one symbol. if return == srcSize, there is only one symbol.
Can also return an error code, which can be tested with FSE_isError(). */ Can also return an error code, which can be tested with FSE_isError(). */
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
/*! FSE_optimalTableLog(): /*! FSE_optimalTableLog():
dynamically downsize 'tableLog' when conditions are met. dynamically downsize 'tableLog' when conditions are met.
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
@return : recommended tableLog (necessarily <= 'maxTableLog') */ @return : recommended tableLog (necessarily <= 'maxTableLog') */
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
/*! FSE_normalizeCount(): /*! FSE_normalizeCount():
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
@return : tableLog, @return : tableLog,
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
/*! FSE_NCountWriteBound(): /*! FSE_NCountWriteBound():
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
Typically useful for allocation purpose. */ Typically useful for allocation purpose. */
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_writeNCount(): /*! FSE_writeNCount():
Compactly save 'normalizedCounter' into 'buffer'. Compactly save 'normalizedCounter' into 'buffer'.
@return : size of the compressed table, @return : size of the compressed table,
or an errorCode, which can be tested using FSE_isError(). */ or an errorCode, which can be tested using FSE_isError(). */
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! Constructor and Destructor of FSE_CTable. /*! Constructor and Destructor of FSE_CTable.
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
void FSE_freeCTable (FSE_CTable* ct); FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
/*! FSE_buildCTable(): /*! FSE_buildCTable():
Builds `ct`, which must be already allocated, using FSE_createCTable(). Builds `ct`, which must be already allocated, using FSE_createCTable().
@return : 0, or an errorCode, which can be tested using FSE_isError() */ @return : 0, or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_compress_usingCTable(): /*! FSE_compress_usingCTable():
Compress `src` using `ct` into `dst` which must be already allocated. Compress `src` using `ct` into `dst` which must be already allocated.
@return : size of compressed data (<= `dstCapacity`), @return : size of compressed data (<= `dstCapacity`),
or 0 if compressed data could not fit into `dst`, or 0 if compressed data could not fit into `dst`,
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
/*! /*!
Tutorial : Tutorial :
@ -223,25 +249,25 @@ If there is an error, the function will return an ErrorCode (which can be tested
@return : size read from 'rBuffer', @return : size read from 'rBuffer',
or an errorCode, which can be tested using FSE_isError(). or an errorCode, which can be tested using FSE_isError().
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
/*! Constructor and Destructor of FSE_DTable. /*! Constructor and Destructor of FSE_DTable.
Note that its size depends on 'tableLog' */ Note that its size depends on 'tableLog' */
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_DTable* FSE_createDTable(unsigned tableLog); FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
void FSE_freeDTable(FSE_DTable* dt); FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
/*! FSE_buildDTable(): /*! FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable(). Builds 'dt', which must be already allocated, using FSE_createDTable().
return : 0, or an errorCode, which can be tested using FSE_isError() */ return : 0, or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_decompress_usingDTable(): /*! FSE_decompress_usingDTable():
Decompress compressed source `cSrc` of size `cSrcSize` using `dt` Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
into `dst` which must be already allocated. into `dst` which must be already allocated.
@return : size of regenerated data (necessarily <= `dstCapacity`), @return : size of regenerated data (necessarily <= `dstCapacity`),
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
/*! /*!
Tutorial : Tutorial :

10
lib/compress/fse_compress.c
View File

@ -201,8 +201,6 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
} }
static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
unsigned writeIsSafe) unsigned writeIsSafe)
@ -258,16 +256,16 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
bitStream >>= 16; bitStream >>= 16;
bitCount -= 16; bitCount -= 16;
} } } }
{ short count = normalizedCounter[charnum++]; { int count = normalizedCounter[charnum++];
const short max = (short)((2*threshold-1)-remaining); int const max = (2*threshold-1)-remaining;
remaining -= FSE_abs(count); remaining -= count < 0 ? -count : count;
if (remaining<1) return ERROR(GENERIC);
count++; /* +1 for extra accuracy */ count++; /* +1 for extra accuracy */
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
bitStream += count << bitCount; bitStream += count << bitCount;
bitCount += nbBits; bitCount += nbBits;
bitCount -= (count<max); bitCount -= (count<max);
previous0 = (count==1); previous0 = (count==1);
if (remaining<1) return ERROR(GENERIC);
while (remaining<threshold) nbBits--, threshold>>=1; while (remaining<threshold) nbBits--, threshold>>=1;
} }
if (bitCount>16) { if (bitCount>16) {