lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-08 17:22:10 +03:00
Code Activity

created ZSTD_compress2() and ZSTD_compressStream2()

Browse Source 
ZSTD_compress_generic() is renamed ZSTD_compressStream2().

Note that, for the time being,
the "stable" API and advanced one use different parameter planes :
setting parameters using the advanced API does not influence ZSTD_compressStream()
and using ZSTD_initCStream() does not influence parameters for ZSTD_compressStream2().
This commit is contained in:
Yann Collet
2018-11-30 11:16:26 -08:00
parent d3a0c71259
commit d8e215cbee
12 changed files with 133 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
tests/fullbench.c
View File

@@ -171,17 +171,8 @@ local_ZSTD_compress_generic_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
return buffOut.pos;
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
}
static size_t
@@ -198,8 +189,8 @@ local_ZSTD_compress_generic_continue(const void* src, size_t srcSize,
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end);
return buffOut.pos;
}
@@ -208,18 +199,9 @@ local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
while (ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
return buffOut.pos;
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
}
static size_t
@@ -237,8 +219,8 @@ local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize,
buffIn.src = src;
buffIn.size = srcSize;
buffIn.pos = 0;
ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
while(ZSTD_compress_generic(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue);
while(ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {}
return buffOut.pos;
}

2
tests/fuzz/simple_round_trip.c
View File

@@ -42,7 +42,7 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
FUZZ_setRandomParameters(cctx, srcSize, &seed);
err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
err = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(err);
FUZZ_ASSERT(err == 0);
cSize = out.pos;

8
tests/fuzz/stream_round_trip.c
View File

@@ -72,7 +72,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
case 1: /* fall-though */
case 2: {
size_t const ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush);
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
FUZZ_ZASSERT(ret);
if (ret == 0)
mode = -1;
@@ -80,7 +80,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
}
case 3: {
size_t ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(ret);
/* Reset the compressor when the frame is finished */
if (ret == 0) {
@@ -95,7 +95,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
}
default: {
size_t const ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
FUZZ_ZASSERT(ret);
mode = -1;
}
@@ -108,7 +108,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
for (;;) {
ZSTD_inBuffer in = {NULL, 0, 0};
ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(ret);
dst += out.pos;

16
tests/fuzzer.c
View File

@@ -237,7 +237,7 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig
ZSTD_inBuffer in = { inBuffer, inSize, 0 };
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, compressionLevel) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) );
while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {}
while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
ZSTD_freeCCtx(cctx);
DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ",
nbThreads, compressionLevel);
@@ -257,8 +257,8 @@ static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsig
ZSTD_inBuffer in = { inBuffer, inSize, 0 };
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, compressionLevel) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbWorkers, nbThreads) );
CHECK_Z( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue) );
while ( ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end) ) {}
CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
ZSTD_freeCCtx(cctx);
DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ",
nbThreads, compressionLevel);
@@ -521,7 +521,7 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
CHECK_EQ(value, ZSTD_HASHLOG_MIN);
/* Start a compression job */
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_compressionLevel, &value));
CHECK_EQ(value, 7);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_p_hashLog, &value));
@@ -1255,7 +1255,7 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, compressionLevel) );
{ ZSTD_inBuffer in = { CNBuffer, srcSize, 0 };
ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
size_t const compressionResult = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
size_t const compressionResult = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, out.pos);
if (ZSTD_isError(compressionResult)) goto _output_error;
if (out.pos != cSize_1pass) goto _output_error;
@@ -1276,7 +1276,7 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_windowLog, 18) );
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
cSize = out.pos;
@@ -1293,7 +1293,7 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_compressionLevel, 2) );
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
if (out.pos != cSize) goto _output_error; /* must result in same compressed result, hence same size */
@@ -1313,7 +1313,7 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
if (result != 0) goto _output_error;
if (in.pos != in.size) goto _output_error;
cSize = out.pos;

29
tests/paramgrill.c
View File

@@ -920,30 +920,13 @@ static size_t local_initDCtx(void* payload) {
/* additional argument is just the context */
static size_t local_defaultCompress(
const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstSize,
void* addArgs) {
size_t moreToFlush = 1;
ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
ZSTD_inBuffer in;
ZSTD_outBuffer out;
in.src = srcBuffer;
in.size = srcSize;
in.pos = 0;
out.dst = dstBuffer;
out.size = dstSize;
out.pos = 0;
const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstSize,
void* addArgs)
{
ZSTD_CCtx* cctx = (ZSTD_CCtx*)addArgs;
assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */
while (moreToFlush) {
if(out.pos == out.size) {
return (size_t)-ZSTD_error_dstSize_tooSmall;
}
moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
if (ZSTD_isError(moreToFlush)) {
return moreToFlush;
}
}
return out.pos;
return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize);
}
/* additional argument is just the context */

4
tests/roundTripCrash.c
View File

@@ -85,7 +85,7 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 };
ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 };
ZSTD_outBuffer outBuffer = { compressedBuff, compressedBuffCapacity, 0 };
static const int maxClevel = 19;
size_t const hashLength = MIN(128, srcBuffSize);
@@ -101,7 +101,7 @@ static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity
/* Apply parameters */
CHECK_Z( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) );
CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) );
CHECK_Z (ZSTD_compressStream2(cctx, &outBuffer, &inBuffer, ZSTD_e_end) );
ZSTD_freeCCtxParams(cctxParams);
ZSTD_freeCCtx(cctx);

28
tests/zstreamtest.c
View File

@@ -175,11 +175,11 @@ static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
size_t cret;
do {
ZSTD_outBuffer cout = {compressed, sizeof(compressed), 0};
ZSTD_inBuffer din = {compressed, 0, 0};
ZSTD_outBuffer dout = {uncompressed, 0, 0};
ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
ZSTD_inBuffer din = { compressed, 0, 0 };
ZSTD_outBuffer dout = { uncompressed, 0, 0 };
cret = ZSTD_compress_generic(cctx, &cout, &cin, endOp);
cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
if (ZSTD_isError(cret))
return cret;
@@ -686,7 +686,7 @@ static int basicUnitTests(U32 seed, double compressibility)
inBuff.size = size;
inBuff.pos = 0;
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
CHECK_Z(ZSTD_compress_generic(cctx, &outBuff, &inBuff, ZSTD_e_end));
CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
if (inBuff.pos != inBuff.size) goto _output_error;
{ ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
@@ -746,7 +746,7 @@ static int basicUnitTests(U32 seed, double compressibility)
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
cSize = outBuff.pos;
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
@@ -770,14 +770,14 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
}
DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compress_generic : ", testNb++);
DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
cSize = outBuff.pos;
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
@@ -885,7 +885,7 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
ZSTD_freeCDict(cdict);
}
@@ -1086,7 +1086,7 @@ static int basicUnitTests(U32 seed, double compressibility)
const size_t kSmallBlockSize = sizeof(testBuffer);
ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
CHECK_Z(ZSTD_compress_generic(zc, &out, &in, ZSTD_e_flush));
CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
CHECK(in.pos != in.size, "input not fully consumed");
remainingInput -= kSmallBlockSize;
}
@@ -1094,7 +1094,7 @@ static int basicUnitTests(U32 seed, double compressibility)
for (offset = 1024; offset >= 0; offset -= 128) {
ZSTD_inBuffer in = {dictionary.start + offset, 128, 0};
ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
CHECK_Z(ZSTD_compress_generic(zc, &out, &in, flush));
CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
CHECK(in.pos != in.size, "input not fully consumed");
}
/* Ensure decompression works */
@@ -1956,7 +1956,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
outBuff.size = outBuff.pos + dstBuffSize;
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos), (U32)flush, (U32)outBuff.pos);
@@ -1973,10 +1973,10 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest,
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
outBuff.size = outBuff.pos + adjustedDstSize;
DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (U32)adjustedDstSize);
remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (U32)outBuff.pos);
CHECK( ZSTD_isError(remainingToFlush),
"ZSTD_compress_generic w/ ZSTD_e_end error : %s",
"ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
ZSTD_getErrorName(remainingToFlush) );
} }
crcOrig = XXH64_digest(&xxhState);