lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-07-30 22:23:13 +03:00
Code Activity

Updated fullbench tests

Browse Source
This commit is contained in:
Yann Collet
2015-01-24 13:31:55 +01:00
parent 18a68714d0
commit 759433d679
4 changed files with 77 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
lib/zstd.c
View File

@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
//U32 limit = 4 * BLOCKSIZE; //U32 limit = 4 * BLOCKSIZE;
//const U32 updateRate = 2 * BLOCKSIZE; //const U32 updateRate = 2 * BLOCKSIZE;
// Init /* Init */
if (ctx->base==NULL) ctx->base = src, ctx->current=0; if (ctx->base==NULL) ctx->base = src, ctx->current=0;
if (src != ctx->base + ctx->current) // not contiguous if (src != ctx->base + ctx->current) /* not contiguous */
{ {
ZSTD_resetCCtx(ctx); ZSTD_resetCCtx(ctx);
ctx->base = src; ctx->base = src;
@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
} }
*/ */
// compress /* compress */
if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize); cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
if (cSize == 0) if (cSize == 0)
{ {
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
} }
else else
@ -1084,7 +1084,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
BYTE* const ostart = dst; BYTE* const ostart = dst;
BYTE* op = ostart; BYTE* op = ostart;
// Header /* Header */
{ {
size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize); size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
if(ZSTD_isError(headerSize)) return headerSize; if(ZSTD_isError(headerSize)) return headerSize;
@ -1092,7 +1092,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
maxDstSize -= headerSize; maxDstSize -= headerSize;
} }
// Compression /* Compression */
{ {
size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize); size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
@ -1100,7 +1100,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
maxDstSize -= cSize; maxDstSize -= cSize;
} }
// Close frame /* Close frame */
{ {
size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize); size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize);
if(ZSTD_isError(endSize)) return endSize; if(ZSTD_isError(endSize)) return endSize;

10
programs/Makefile
View File

@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen
./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID) ./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID)
./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID) ./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID)
test-fullbench: fullbench test-fullbench: fullbench datagen
./fullbench -i1 ./fullbench -i1
./datagen -P0 -g516K > tmp
./fullbench -i1 tmp
@rm tmp
test-fullbench32: fullbench32 test-fullbench32: fullbench32 datagen
./fullbench32 -i1 ./fullbench32 -i1
./datagen -P0 -g516K > tmp
./fullbench32 -i1 tmp
@rm tmp
test-fuzzer: fuzzer test-fuzzer: fuzzer
./fuzzer ./fuzzer

65
programs/datagen.c
View File

@ -56,6 +56,24 @@
#endif #endif
/**************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# ifdef __MINGW32__
int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
# endif
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#endif
/************************************** /**************************************
* Constants * Constants
**************************************/ **************************************/
@ -70,7 +88,6 @@
#define CDG_SIZE_DEFAULT (64 KB) #define CDG_SIZE_DEFAULT (64 KB)
#define CDG_SEED_DEFAULT 0 #define CDG_SEED_DEFAULT 0
#define CDG_COMPRESSIBILITY_DEFAULT 50 #define CDG_COMPRESSIBILITY_DEFAULT 50
#define CDG_LITDENSITY_DEFAULT 12
#define PRIME1 2654435761U #define PRIME1 2654435761U
#define PRIME2 2246822519U #define PRIME2 2246822519U
@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src)
{ {
U32 rand32 = *src; U32 rand32 = *src;
rand32 *= PRIME1; rand32 *= PRIME1;
rand32 += PRIME2; rand32 ^= PRIME2;
rand32 = CDG_rotl32(rand32, 13); rand32 = CDG_rotl32(rand32, 13);
*src = rand32; *src = rand32;
return rand32; return rand32;
@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src)
#define LTSIZE 8192 #define LTSIZE 8192
#define LTMASK (LTSIZE-1) #define LTMASK (LTSIZE-1)
static const char firstChar = '(';
static const char lastChar = '}';
static void* CDG_createLiteralDistrib(double ld) static void* CDG_createLiteralDistrib(double ld)
{ {
char* lt = malloc(LTSIZE); BYTE* lt = malloc(LTSIZE);
U32 i = 0; U32 i = 0;
char character = '0'; BYTE character = '0';
BYTE firstChar = '(';
BYTE lastChar = '}';
if (ld==0.0)
{
character = 0;
firstChar = 0;
lastChar =255;
}
while (i<LTSIZE) while (i<LTSIZE)
{ {
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1; U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
@ -146,8 +169,10 @@ static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba
U32 pos=1; U32 pos=1;
U32 genBlockSize = 128 KB; U32 genBlockSize = 128 KB;
void* ldctx = CDG_createLiteralDistrib(litProba); void* ldctx = CDG_createLiteralDistrib(litProba);
FILE* fout = stdout;
/* Build initial prefix */ /* init */
SET_BINARY_MODE(stdout);
fullbuff[0] = CDG_genChar(seed, ldctx); fullbuff[0] = CDG_genChar(seed, ldctx);
while (pos<32 KB) while (pos<32 KB)
{ {
@ -206,11 +231,8 @@ static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba
} }
} }
/* output datagen */ /* output generated data */
pos=0; fwrite(buff, 1, genBlockSize, fout);
for (;pos+512<=genBlockSize;pos+=512)
printf("%512.512s", buff+pos);
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
/* Regenerate prefix */ /* Regenerate prefix */
memcpy(fullbuff, buff + 96 KB, 32 KB); memcpy(fullbuff, buff + 96 KB, 32 KB);
} }
@ -238,8 +260,8 @@ static int CDG_usage(char* programName)
int main(int argc, char** argv) int main(int argc, char** argv)
{ {
int argNb; int argNb;
U32 proba = CDG_COMPRESSIBILITY_DEFAULT; double proba = (double)CDG_COMPRESSIBILITY_DEFAULT / 100;
U32 litProba = CDG_LITDENSITY_DEFAULT; double litProba = proba / 3.6;
U64 size = CDG_SIZE_DEFAULT; U64 size = CDG_SIZE_DEFAULT;
U32 seed = CDG_SEED_DEFAULT; U32 seed = CDG_SEED_DEFAULT;
char* programName; char* programName;
@ -290,25 +312,28 @@ int main(int argc, char** argv)
break; break;
case 'P': case 'P':
argument++; argument++;
proba=0; proba=0.0;
while ((*argument>='0') && (*argument<='9')) while ((*argument>='0') && (*argument<='9'))
{ {
proba *= 10; proba *= 10;
proba += *argument - '0'; proba += *argument - '0';
argument++; argument++;
} }
if (proba>100) proba=100; if (proba>100.) proba=100.;
proba /= 100.;
litProba = proba / 4.;
break; break;
case 'L': case 'L':
argument++; argument++;
litProba=0; litProba=0.;
while ((*argument>='0') && (*argument<='9')) while ((*argument>='0') && (*argument<='9'))
{ {
litProba *= 10; litProba *= 10;
litProba += *argument - '0'; litProba += *argument - '0';
argument++; argument++;
} }
if (litProba>100) litProba=100; if (litProba>100.) litProba=100.;
litProba /= 100.;
break; break;
case 'v': case 'v':
displayLevel = 4; displayLevel = 4;
@ -324,9 +349,9 @@ int main(int argc, char** argv)
DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION); DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
DISPLAYLEVEL(3, "Seed = %u \n", seed); DISPLAYLEVEL(3, "Seed = %u \n", seed);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba); if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
CDG_generate(size, &seed, ((double)proba) / 100, ((double)litProba) / 100); CDG_generate(size, &seed, proba, litProba);
return 0; return 0;
} }

18
programs/fullbench.c
View File

@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
{ {
blockProperties_t bp; blockProperties_t bp;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize); ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block compressed size
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
free(dstBuff);
free(buff2);
return 0;
}
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3; g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3;
memcpy(buff2, dstBuff+7, g_cSize); memcpy(buff2, dstBuff+7, g_cSize);
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed //srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize); ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ip += 4; // Jump magic Number ip += 4; // Jump magic Number
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
free(dstBuff);
free(buff2);
return 0;
}
iend = ip + 3 + blockSize; // Get end of first block iend = ip + 3 + blockSize; // Get end of first block
ip += 3; // jump first block header ip += 3; // jump first block header
ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header
@ -450,6 +465,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
default : ; default : ;
} }
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
for (loopNb = 1; loopNb <= nbIterations; loopNb++) for (loopNb = 1; loopNb <= nbIterations; loopNb++)
{ {
double averageTime; double averageTime;
@ -457,7 +474,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
U32 nbRounds=0; U32 nbRounds=0;
DISPLAY("%2i- %-30.30s : \r", loopNb, benchName); DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
milliTime = BMK_GetMilliStart(); milliTime = BMK_GetMilliStart();
while(BMK_GetMilliStart() == milliTime); while(BMK_GetMilliStart() == milliTime);