lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-01 09:47:01 +03:00
Code Activity

quick first zstdhc version

Browse Source
This commit is contained in:
Yann Collet
2015-10-22 15:31:46 +01:00
parent 353c5d26cf
commit f3eca25322
9 changed files with 866 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
programs/Makefile
View File

@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
VERSION?= v0.2.0
VERSION?= 0.3.0
DESTDIR?=
PREFIX ?= /usr/local
@ -58,7 +58,7 @@ default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen
zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c

90
programs/bench.c
View File

@ -23,29 +23,29 @@
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/***************************************
/* **************************************
* Compiler Options
***************************************/
****************************************/
/* Disable some Visual warning messages */
#define _CRT_SECURE_NO_WARNINGS /* fopen */
// Unix Large Files support (>4GB)
/* Unix Large Files support (>4GB) */
#define _FILE_OFFSET_BITS 64
#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
# define _LARGEFILE_SOURCE
#elif ! defined(__LP64__) // No point defining Large file for 64 bit
#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
# define _LARGEFILE64_SOURCE
#endif
// S_ISREG & gettimeofday() are not supported by MSVC
/* S_ISREG & gettimeofday() are not supported by MSVC */
#if defined(_MSC_VER) || defined(_WIN32)
# define BMK_LEGACY_TIMER 1
#endif
/**************************************
/* *************************************
* Includes
**************************************/
***************************************/
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> // fprintf, fopen, ftello64
@ -59,40 +59,23 @@
# include <sys/time.h> // gettimeofday
#endif
#include "mem.h"
#include "zstd.h"
#include "zstdhc.h"
#include "xxhash.h"
/**************************************
/* *************************************
* Compiler specifics
**************************************/
***************************************/
#if !defined(S_ISREG)
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
/**************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
/* *************************************
* Constants
**************************************/
***************************************/
#define NBLOOPS 3
#define TIMELOOP 2500
@ -108,15 +91,15 @@ static U32 prime1 = 2654435761U;
static U32 prime2 = 2246822519U;
/**************************************
/* *************************************
* Macros
**************************************/
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
/**************************************
/* *************************************
* Benchmark Parameters
**************************************/
***************************************/
static int nbIterations = NBLOOPS;
static size_t g_blockSize = 0;
@ -133,9 +116,9 @@ void BMK_SetBlockSize(size_t blockSize)
}
/*********************************************************
/* ********************************************************
* Private functions
*********************************************************/
**********************************************************/
#if defined(BMK_LEGACY_TIMER)
@ -176,10 +159,9 @@ static int BMK_GetMilliSpan( int nTimeStart )
}
/*********************************************************
/* ********************************************************
* Data generator
*********************************************************/
**********************************************************/
/* will hopefully be converted into ROL instruction by compiler */
static U32 BMK_rotl32(unsigned val32, unsigned nbBits) { return((val32 << nbBits) | (val32 >> (32 - nbBits))); }
@ -234,9 +216,9 @@ static void BMK_datagen(void* buffer, size_t bufferSize, double proba, U32 seed)
}
/*********************************************************
/* ********************************************************
* Bench functions
*********************************************************/
**********************************************************/
typedef struct
{
char* srcPtr;
@ -249,6 +231,14 @@ typedef struct
} blockParam_t;
typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel);
static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel)
{
(void)compressionLevel;
return ZSTD_compress(dst, maxDstSize, src, srcSize);
}
#define MIN(a,b) (a<b ? a : b)
static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
@ -259,10 +249,12 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
compressor_t compressor;
U64 crcOrig;
/* Init */
(void)cLevel;
if (cLevel <= 1) compressor = local_compress_fast;
else compressor = ZSTD_HC_compress;
/* Memory allocation & restrictions */
if (!compressedBuffer || !resultBuffer || !blockTable)
@ -328,8 +320,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
{
for (blockNb=0; blockNb<nbBlocks; blockNb++)
blockTable[blockNb].cSize = ZSTD_compress(blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
blockTable[blockNb].cSize = compressor(blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel);
nbLoops++;
}
milliTime = BMK_GetMilliSpan(milliTime);
@ -364,13 +355,14 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck)
{
unsigned i;
unsigned u;
unsigned eBlockSize = MIN(65536*2, blockSize);
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
for (i=0; i<srcSize; i++)
for (u=0; u<srcSize; u++)
{
if (((BYTE*)srcBuffer)[i] != ((BYTE*)resultBuffer)[i])
if (((BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u])
{
printf("\nDecoding error at pos %u \n", i);
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
}
}

9
programs/fileio.c
View File

@ -116,12 +116,6 @@ static const unsigned FIO_blockHeaderSize = 3;
#define CACHELINE 64
/**************************************
* Complex types
**************************************/
typedef enum { bt_compressed, bt_raw, bt_rle, bt_crc } bType_t;
/**************************************
* Macros
**************************************/
@ -218,7 +212,7 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
}
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename)
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, unsigned cLevel)
{
U64 filesize = 0;
U64 compressedfilesize = 0;
@ -236,6 +230,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* Init */
(void)cLevel;
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
/* Allocate Memory */

20
programs/fileio.h
View File

@ -29,7 +29,7 @@ extern "C" {
#endif
/**************************************
/* *************************************
* Special i/o constants
**************************************/
#define nullString "null"
@ -42,28 +42,28 @@ extern "C" {
#endif
/**************************************
/* *************************************
* Parameters
**************************************/
***************************************/
void FIO_overwriteMode(void);
void FIO_setNotificationLevel(unsigned level);
/**************************************
/* *************************************
* Stream/File functions
**************************************/
unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename);
***************************************/
unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, unsigned cLevel);
unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename);
/*
/**
FIO_compressFilename :
result : size of compressed file
@result : size of compressed file
FIO_decompressFilename :
result : size of regenerated file
@result : size of regenerated file
*/
#if defined (__cplusplus)
}
#endif
#endif

18
programs/zstdcli.c
View File

@ -169,6 +169,7 @@ int main(int argc, char** argv)
main_pause=0;
unsigned fileNameStart = 0;
unsigned nbFiles = 0;
unsigned cLevel = 0;
const char* programName = argv[0];
const char* inFileName = NULL;
const char* outFileName = NULL;
@ -217,6 +218,19 @@ int main(int argc, char** argv)
while (argument[0]!=0)
{
/* compression Level */
if ((*argument>='0') && (*argument<='9'))
{
cLevel = 0;
while ((*argument >= '0') && (*argument <= '9'))
{
cLevel *= 10;
cLevel += *argument - '0';
argument++;
}
continue;
}
switch(argument[0])
{
/* Display help */
@ -307,7 +321,7 @@ int main(int argc, char** argv)
if (!strcmp(inFileName, stdinmark) && IS_CONSOLE(stdin) ) return badusage(programName);
/* Check if benchmark is selected */
if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, 0); goto _end; }
if (bench) { BMK_benchFiles(argv+fileNameStart, nbFiles, cLevel); goto _end; }
/* No output filename ==> try to select one automatically (when possible) */
while (!outFileName)
@ -351,7 +365,7 @@ int main(int argc, char** argv)
if (decode)
FIO_decompressFilename(outFileName, inFileName);
else
FIO_compressFilename(outFileName, inFileName);
FIO_compressFilename(outFileName, inFileName, cLevel);
_end:
if (main_pause) waitEnter();