diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index dd6f9d7ab..ecb2b622c 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1421,7 +1421,8 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx, ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const ZSTD_Sequence* inSeqs, size_t nbSequences, - const void* literals, size_t litSize, size_t srcSize); + const void* literals, size_t litSize, size_t litCapacity, + size_t decompressedSize);

This is a variant of ZSTD_compressSequences() which, instead of receiving (src,srcSize) as input parameter, receives (literals,litSize), aka all the literals, already extracted and laid out into a single continuous buffer. @@ -1434,7 +1435,8 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, - Not compatible with frame checksum, which must be disabled - If any block is incompressible, will fail and return an error - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error. - - the buffer @literals must be larger than @litSize by at least 8 bytes. + - the buffer @literals must have a size @litCapacity which is larger than @litSize by at least 8 bytes. + - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error. @return : final compressed size, or a ZSTD error code.


diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b5831650f..b5e88325d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -7306,7 +7306,8 @@ size_t ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* literals, size_t litSize, size_t srcSize) + const void* literals, size_t litSize, size_t litCapacity, + size_t decompressedSize) { BYTE* op = (BYTE*)dst; size_t cSize = 0; @@ -7314,7 +7315,10 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, /* Transparent initialization stage, same as compressStream2() */ DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals (dstCapacity=%zu)", dstCapacity); assert(cctx != NULL); - FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); + if (litCapacity < litSize) { + RETURN_ERROR(workSpace_tooSmall, "literals buffer is not large enough: must be at least 8 bytes larger than litSize (risk of read out-of-bound)"); + } + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, decompressedSize), "CCtx initialization failed"); if (cctx->appliedParams.blockDelimiters == ZSTD_sf_noBlockDelimiters) { RETURN_ERROR(frameParameter_unsupported, "This mode is only compatible with explicit delimiters"); @@ -7328,7 +7332,7 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, /* Begin writing output, starting with frame header */ { size_t const frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, - &cctx->appliedParams, srcSize, cctx->dictID); + &cctx->appliedParams, decompressedSize, cctx->dictID); op += frameHeaderSize; assert(frameHeaderSize <= dstCapacity); dstCapacity -= frameHeaderSize; @@ -7339,7 +7343,7 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, { size_t const cBlocksSize = ZSTD_compressSequencesAndLiterals_internal(cctx, op, dstCapacity, inSeqs, inSeqsSize, - literals, litSize, srcSize); + literals, litSize, decompressedSize); FORWARD_IF_ERROR(cBlocksSize, "Compressing blocks failed!"); cSize += cBlocksSize; assert(cBlocksSize <= dstCapacity); diff --git a/lib/zstd.h b/lib/zstd.h index feb1ae17a..36668b85b 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -1678,14 +1678,16 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx, * - Not compatible with frame checksum, which must be disabled * - If any block is incompressible, will fail and return an error * - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error. - * - the buffer @literals must be larger than @litSize by at least 8 bytes. + * - the buffer @literals must have a size @litCapacity which is larger than @litSize by at least 8 bytes. + * - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error. * @return : final compressed size, or a ZSTD error code. */ ZSTDLIB_STATIC_API size_t ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const ZSTD_Sequence* inSeqs, size_t nbSequences, - const void* literals, size_t litSize, size_t srcSize); + const void* literals, size_t litSize, size_t litCapacity, + size_t decompressedSize); /*! ZSTD_writeSkippableFrame() : diff --git a/tests/fullbench.c b/tests/fullbench.c index 05d6bf027..5683eca25 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -612,7 +612,7 @@ local_compressSequencesAndLiterals(const void* input, size_t inputSize, void* payload) { const char* ip = input; - size_t srcSize = MEM_read32(ip); + size_t decompressedSize = MEM_read32(ip); size_t nbSeqs = MEM_read32(ip+=4); size_t nbLiterals = MEM_read32(ip+=4); const ZSTD_Sequence* seqs = (const ZSTD_Sequence*)(const void*)(ip+=4); @@ -625,7 +625,7 @@ local_compressSequencesAndLiterals(const void* input, size_t inputSize, assert(12 + nbSeqs * sizeof(ZSTD_Sequence) + nbLiterals == inputSize); (void)inputSize; (void)payload; - return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals, srcSize); + return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals, nbLiterals + 8, decompressedSize); } static PrepResult prepConvertSequences(const void* src, size_t srcSize, int cLevel) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 335fbd5c4..6d165aea4 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -3909,35 +3909,35 @@ static int basicUnitTests(U32 const seed, double compressibility) FUZ_transferLiterals(litBuffer, decompressSize, CNBuffer, srcSize, seqs, nbSeqs); /* not enough literals: must fail */ - compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, srcSize); + compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, decompressSize, srcSize); if (!ZSTD_isError(compressedSize)) { DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: not enough literals provided\n"); goto _output_error; } /* too many literals: must fail */ - compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, srcSize); + compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, decompressSize, srcSize); if (!ZSTD_isError(compressedSize)) { DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: too many literals provided\n"); goto _output_error; } /* srcSize too large: must fail */ - compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, srcSize+1); + compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize+1); if (!ZSTD_isError(compressedSize)) { DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too large\n"); goto _output_error; } /* srcSize too small: must fail */ - compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, srcSize-1); + compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize-1); if (!ZSTD_isError(compressedSize)) { DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too small\n"); goto _output_error; } /* correct amount of literals: should compress successfully */ - compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, srcSize); + compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize); if (ZSTD_isError(compressedSize)) { DISPLAY("Error in ZSTD_compressSequencesAndLiterals()\n"); goto _output_error;