lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

updated fse version

Browse Source 
feature minor refactoring (removing FSE_abs())
also : fix a few minor issues recently introduced in examples
This commit is contained in:
Yann Collet
2017-02-15 12:00:03 -08:00
parent c09d16ba8c
commit 4596037042
6 changed files with 80 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
examples/dictionary_decompression.c
View File

@ -13,6 +13,7 @@
#include <string.h> // strerror
#include <errno.h> // errno
#include <sys/stat.h> // stat
#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize
#include <zstd.h> // presumes zstd library is installed

6
examples/simple_compression.c
View File

@ -102,7 +102,7 @@ static void compress_orDie(const char* fname, const char* oname)
}
static const char* createOutFilename_orDie(const char* filename)
static char* createOutFilename_orDie(const char* filename)
{
size_t const inL = strlen(filename);
size_t const outL = inL + 5;
@ -110,7 +110,7 @@ static const char* createOutFilename_orDie(const char* filename)
memset(outSpace, 0, outL);
strcat(outSpace, filename);
strcat(outSpace, ".zst");
return (const char*)outSpace;
return (char*)outSpace;
}
int main(int argc, const char** argv)
@ -125,7 +125,7 @@ int main(int argc, const char** argv)
return 1;
}
const char* const outFilename = createOutFilename_orDie(inFilename);
char* const outFilename = createOutFilename_orDie(inFilename);
compress_orDie(inFilename, outFilename);
free(outFilename);
return 0;

21
examples/simple_decompression.c
View File

@ -6,17 +6,16 @@
* LICENSE-examples file in the root directory of this source tree.
*/
#include <stdlib.h> // malloc, exit
#include <stdio.h> // printf
#include <string.h> // strerror
#include <errno.h> // errno
#include <sys/stat.h> // stat
#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize
#include <zstd.h> // presumes zstd library is installed
static off_t fsize_X(const char *filename)
static off_t fsize_orDie(const char *filename)
{
struct stat st;
if (stat(filename, &st) == 0) return st.st_size;
@ -25,7 +24,7 @@ static off_t fsize_X(const char *filename)
exit(1);
}
static FILE* fopen_X(const char *filename, const char *instruction)
static FILE* fopen_orDie(const char *filename, const char *instruction)
{
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
@ -34,7 +33,7 @@ static FILE* fopen_X(const char *filename, const char *instruction)
exit(2);
}
static void* malloc_X(size_t size)
static void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
if (buff) return buff;
@ -43,11 +42,11 @@ static void* malloc_X(size_t size)
exit(3);
}
static void* loadFile_X(const char* fileName, size_t* size)
static void* loadFile_orDie(const char* fileName, size_t* size)
{
off_t const buffSize = fsize_X(fileName);
FILE* const inFile = fopen_X(fileName, "rb");
void* const buffer = malloc_X(buffSize);
off_t const buffSize = fsize_orDie(fileName);
FILE* const inFile = fopen_orDie(fileName, "rb");
void* const buffer = malloc_orDie(buffSize);
size_t const readSize = fread(buffer, 1, buffSize, inFile);
if (readSize != (size_t)buffSize) {
printf("fread: %s : %s \n", fileName, strerror(errno));
@ -62,13 +61,13 @@ static void* loadFile_X(const char* fileName, size_t* size)
static void decompress(const char* fname)
{
size_t cSize;
void* const cBuff = loadFile_X(fname, &cSize);
void* const cBuff = loadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==0) {
printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
exit(5);
}
void* const rBuff = malloc_X((size_t)rSize);
void* const rBuff = malloc_orDie((size_t)rSize);
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);

24
lib/common/entropy_common.c
View File

@ -43,6 +43,12 @@
#include "huf.h"
/*-****************************************
* Version
******************************************/
unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
/*-****************************************
* FSE Error Management
******************************************/
@ -62,8 +68,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*-**************************************************************
* FSE NCount encoding-decoding
****************************************************************/
static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
const void* headerBuffer, size_t hbSize)
{
@ -117,21 +121,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
} else {
bitStream >>= 2;
} }
{ short const max = (short)((2*threshold-1)-remaining);
short count;
{ int const max = (2*threshold-1) - remaining;
int count;
if ((bitStream & (threshold-1)) < (U32)max) {
count = (short)(bitStream & (threshold-1));
bitCount += nbBits-1;
count = bitStream & (threshold-1);
bitCount += nbBits-1;
} else {
count = (short)(bitStream & (2*threshold-1));
count = bitStream & (2*threshold-1);
if (count >= threshold) count -= max;
bitCount += nbBits;
bitCount += nbBits;
}
count--; /* extra accuracy */
remaining -= FSE_abs(count);
normalizedCounter[charnum++] = count;
remaining -= count < 0 ? -count : count; /* -1 means +1 */
normalizedCounter[charnum++] = (short)count;
previous0 = !count;
while (remaining < threshold) {
nbBits--;

70
lib/common/fse.h
View File

@ -45,6 +45,32 @@ extern "C" {
#include <stddef.h> /* size_t, ptrdiff_t */
/*-*****************************************
* FSE_PUBLIC_API : control library symbols visibility
******************************************/
#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
# define FSE_PUBLIC_API __declspec(dllexport)
#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define FSE_PUBLIC_API
#endif
/*------ Version ------*/
#define FSE_VERSION_MAJOR 0
#define FSE_VERSION_MINOR 9
#define FSE_VERSION_RELEASE 0
#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
#define FSE_QUOTE(str) #str
#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */
/*-****************************************
* FSE simple functions
******************************************/
@ -56,8 +82,8 @@ extern "C" {
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
*/
size_t FSE_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/*! FSE_decompress():
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
@ -69,18 +95,18 @@ size_t FSE_compress(void* dst, size_t dstCapacity,
Why ? : making this distinction requires a header.
Header management is intentionally delegated to the user layer, which can better manage special cases.
*/
size_t FSE_decompress(void* dst, size_t dstCapacity,
const void* cSrc, size_t cSrcSize);
FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
const void* cSrc, size_t cSrcSize);
/*-*****************************************
* Tool functions
******************************************/
size_t FSE_compressBound(size_t size); /* maximum compressed size */
FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
/* Error Management */
unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/*-*****************************************
@ -94,7 +120,7 @@ const char* FSE_getErrorName(size_t code); /* provides error code string (usef
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
if FSE_isError(return), it's an error code.
*/
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
/*-*****************************************
@ -127,50 +153,50 @@ or to save and provide normalized distribution using external method.
@return : the count of the most frequent symbol (which is not identified).
if return == srcSize, there is only one symbol.
Can also return an error code, which can be tested with FSE_isError(). */
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
/*! FSE_optimalTableLog():
dynamically downsize 'tableLog' when conditions are met.
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
@return : recommended tableLog (necessarily <= 'maxTableLog') */
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
/*! FSE_normalizeCount():
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
@return : tableLog,
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
/*! FSE_NCountWriteBound():
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
Typically useful for allocation purpose. */
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_writeNCount():
Compactly save 'normalizedCounter' into 'buffer'.
@return : size of the compressed table,
or an errorCode, which can be tested using FSE_isError(). */
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! Constructor and Destructor of FSE_CTable.
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
void FSE_freeCTable (FSE_CTable* ct);
FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
/*! FSE_buildCTable():
Builds `ct`, which must be already allocated, using FSE_createCTable().
@return : 0, or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_compress_usingCTable():
Compress `src` using `ct` into `dst` which must be already allocated.
@return : size of compressed data (<= `dstCapacity`),
or 0 if compressed data could not fit into `dst`,
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
/*!
Tutorial :
@ -223,25 +249,25 @@ If there is an error, the function will return an ErrorCode (which can be tested
@return : size read from 'rBuffer',
or an errorCode, which can be tested using FSE_isError().
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
/*! Constructor and Destructor of FSE_DTable.
Note that its size depends on 'tableLog' */
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_DTable* FSE_createDTable(unsigned tableLog);
void FSE_freeDTable(FSE_DTable* dt);
FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
/*! FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable().
return : 0, or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_decompress_usingDTable():
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
into `dst` which must be already allocated.
@return : size of regenerated data (necessarily <= `dstCapacity`),
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
/*!
Tutorial :

10
lib/compress/fse_compress.c
View File

@ -201,8 +201,6 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
}
static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
unsigned writeIsSafe)
@ -258,16 +256,16 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
bitStream >>= 16;
bitCount -= 16;
} }
{ short count = normalizedCounter[charnum++];
const short max = (short)((2*threshold-1)-remaining);
remaining -= FSE_abs(count);
if (remaining<1) return ERROR(GENERIC);
{ int count = normalizedCounter[charnum++];
int const max = (2*threshold-1)-remaining;
remaining -= count < 0 ? -count : count;
count++; /* +1 for extra accuracy */
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
bitStream += count << bitCount;
bitCount += nbBits;
bitCount -= (count<max);
previous0 = (count==1);
if (remaining<1) return ERROR(GENERIC);
while (remaining<threshold) nbBits--, threshold>>=1;
}
if (bitCount>16) {