lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-01 09:47:01 +03:00
Code Activity

update huff0

Browse Source
This commit is contained in:
Yann Collet
2015-10-18 22:18:32 +01:00
parent 0fef5be1cc
commit b1f3f4b565
19 changed files with 5460 additions and 1785 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
programs/Makefile
View File

@ -30,13 +30,13 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
VERSION?= v0.1.2
VERSION?= v0.2.0
DESTDIR?=
PREFIX ?= /usr/local
CPPFLAGS= -I../lib -I../lib/legacy -DZSTD_VERSION=\"$(VERSION)\" -DZSTD_LEGACY_SUPPORT=1
CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(VERSION)\"
LDFLAGS = -I../lib
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
BINDIR = $(PREFIX)/bin
@ -58,22 +58,22 @@ default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen
zstd : $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c
zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fullbench : $(ZSTDDIR)/zstd.c datagen.c fullbench.c
fullbench : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32: $(ZSTDDIR)/zstd.c datagen.c fullbench.c
fullbench32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : $(ZSTDDIR)/zstd.c datagen.c xxhash.c fuzzer.c
fuzzer : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c xxhash.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(ZSTDDIR)/zstd.c datagen.c xxhash.c fuzzer.c
fuzzer32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c xxhash.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
datagen : datagen.c datagencli.c

209
programs/fileio.c
View File

@ -29,6 +29,16 @@
The license of this file is GPLv2.
*/
/**************************************
* Tuning options
**************************************/
#ifndef ZSTD_LEGACY_SUPPORT
/**LEGACY_SUPPORT :
* decompressor can decode older formats (starting from Zstd 0.1+) */
# define ZSTD_LEGACY_SUPPORT 1
#endif // ZSTD_LEGACY_SUPPORT
/**************************************
* Compiler Options
**************************************/
@ -48,14 +58,19 @@
/**************************************
* Includes
**************************************/
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
#include <errno.h> /* errno */
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
#include <errno.h> /* errno */
#include "mem.h"
#include "fileio.h"
#include "zstd_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_v01.h" /* legacy */
#endif // ZSTD_LEGACY_SUPPORT
/**************************************
* OS-specific Includes
@ -75,25 +90,6 @@
#endif
/**************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
* Constants
**************************************/
@ -217,8 +213,8 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
*pfoutput = fopen( output_filename, "wb" );
}
if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
if ( *pfinput==0 ) EXM_THROW(12, "Pb opening src : %s", input_filename);
if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
}
@ -308,7 +304,70 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
}
#define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
{
size_t outBuffSize = 512 KB;
BYTE* outBuff = (BYTE*)malloc(outBuffSize);
size_t inBuffSize = 128 KB + 8;
BYTE inBuff[128 KB + 8];
BYTE* op = outBuff;
BYTE* const oend = outBuff + outBuffSize;
U64 filesize = 0;
size_t toRead;
size_t sizeCheck;
ZSTDv01_Dctx* dctx = ZSTDv01_createDCtx();
/* init */
if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame");
/* restore header, already read from input */
MEM_writeLE32(inBuff, ZSTDv01_magicNumberLE);
sizeCheck = ZSTDv01_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv01_magicNumberLE)); /* Decode frame header */
if (ZSTDv01_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header");
/* Main decompression Loop */
toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
while (toRead)
{
size_t readSize, decodedSize;
/* Fill input buffer */
if (toRead > inBuffSize)
EXM_THROW(43, "too large block");
readSize = fread(inBuff, 1, toRead, finput);
if (readSize != toRead)
EXM_THROW(44, "Read error");
/* Decode block */
decodedSize = ZSTDv01_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv01_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
filesize += decodedSize;
op += decodedSize;
if (op==oend) op = outBuff;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) );
}
/* prepare for next Block */
toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
}
/* release resources */
free(outBuff);
free(dctx);
return filesize;
}
#endif /* ZSTD_LEGACY_SUPPORT */
unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
BYTE* inBuff, size_t inBuffSize,
BYTE* outBuff, size_t outBuffSize,
@ -357,6 +416,98 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
}
#define MAXHEADERSIZE (FIO_FRAMEHEADERSIZE+3)
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
{
FILE* finput, *foutput;
BYTE* inBuff=NULL;
size_t inBuffSize = 0;
BYTE* outBuff=NULL;
size_t outBuffSize = 0;
U32 blockSize = 128 KB;
U32 wNbBlocks = 4;
U64 filesize = 0;
BYTE* header[MAXHEADERSIZE];
size_t toRead;
size_t sizeCheck;
/* Init */
ZSTD_Dctx* dctx = ZSTD_createDCtx();
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
/* for each frame */
for ( ; ; )
{
/* check magic number -> version */
U32 magicNumber;
toRead = sizeof(ZSTD_magicNumber);;
sizeCheck = fread(header, (size_t)1, toRead, finput);
if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
magicNumber = MEM_readLE32(header);
switch(magicNumber)
{
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
case ZSTDv01_magicNumberLE:
filesize += FIOv01_decompressFrame(foutput, finput);
continue;
#endif /* ZSTD_LEGACY_SUPPORT */
case ZSTD_magicNumber:
break; /* normal case */
default :
EXM_THROW(32, "Error : unknown frame prefix");
}
/* prepare frame decompression, by completing header */
ZSTD_resetDCtx(dctx);
toRead = ZSTD_nextSrcSizeToDecompress(dctx) - sizeof(ZSTD_magicNumber);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(header+sizeof(ZSTD_magicNumber), (size_t)1, toRead, finput);
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, sizeof(ZSTD_magicNumber)+toRead); // Decode frame header
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
/* Here later : blockSize determination */
/* Allocate Memory (if needed) */
{
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
size_t newOutBuffSize = wNbBlocks * blockSize;
if (newInBuffSize > inBuffSize)
{
free(inBuff);
inBuffSize = newInBuffSize;
inBuff = (BYTE*)malloc(inBuffSize);
}
if (newOutBuffSize > outBuffSize)
{
free(outBuff);
outBuffSize = newOutBuffSize;
outBuff = (BYTE*)malloc(outBuffSize);
}
}
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
}
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "Decoded %llu bytes \n", (long long unsigned)filesize);
/* clean */
free(inBuff);
free(outBuff);
ZSTD_freeDCtx(dctx);
fclose(finput);
if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
return filesize;
}
#if 0
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
{
FILE* finput, *foutput;
@ -415,7 +566,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
}
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2,"Decoded %llu bytes \n", (long long unsigned)filesize);
DISPLAYLEVEL(2, "Decoded %llu bytes \n", (long long unsigned)filesize);
/* clean */
free(inBuff);
@ -426,4 +577,4 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
return filesize;
}
#endif

31
programs/fullbench.c
View File

@ -60,6 +60,7 @@
# include <sys/time.h> /* gettimeofday */
#endif
#include "mem.h"
#include "zstd.h"
#include "fse_static.h"
#include "datagen.h"
@ -74,25 +75,6 @@
#endif
/**************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/**************************************
* Constants
**************************************/
@ -243,15 +225,12 @@ size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void*
return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
}
extern size_t ZSTD_decodeLiteralsBlock(void* ctx, void* dst, size_t maxDstSize, const BYTE** litStart, size_t* litSize, const void* src, size_t srcSize);
extern size_t ZSTD_decodeLiteralsBlock(void* ctx, const void* src, size_t srcSize);
size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
U32 ctx[1<<12];
const BYTE* ll;
size_t llSize;
(void)src; (void)srcSize;
ZSTD_decodeLiteralsBlock(ctx, dst, dstSize, &ll, &llSize, buff2, g_cSize);
return (const BYTE*)dst + dstSize - ll;
U32 ctx[40 * 1024];
(void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeLiteralsBlock(ctx, buff2, g_cSize);
}
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)

6
programs/fuzzer.c
View File

@ -199,20 +199,20 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_SrcSize) goto _output_error;
if (result != ERROR(srcSize_wrong)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_SrcSize) goto _output_error;
if (result != ERROR(srcSize_wrong)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
if (!ZSTD_isError(result)) goto _output_error;
if (result != (size_t)-ZSTD_ERROR_SrcSize) goto _output_error;
if (result != ERROR(srcSize_wrong)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);