mirror of
https://github.com/facebook/zstd.git
synced 2025-07-30 22:23:13 +03:00
Rename "External Matchfinder" to "Block-Level Sequence Producer" (#3484)
* change "external matchfinder" to "external sequence producer" * migrate contrib/ to new naming convention * fix contrib build * fix error message * update debug strings * fix def of invalid sequences in zstd.h * nit * update CHANGELOG * fix .gitignore
This commit is contained in:
Elliot Gorokhovsky
committed by
GitHub
GitHub
parent
4aa3bc49da
commit
ff42ed1582
@ -29,7 +29,7 @@ cli: Fix empty string as argument for `--output-dir-*` (#3220, @embg)
|
||||
cli: Fix decompression memory usage reported by `-vv --long` (#3042, @u1f35c, and #3232, @zengyijing)
|
||||
cli: Fix infinite loop when empty input is passed to trainer (#3081, @terrelln)
|
||||
cli: Fix `--adapt` doesn't work when `--no-progress` is also set (#3354, @terrelln)
|
||||
api: Support for External matchfinder (#3333, @embg)
|
||||
api: Support for Block-Level Sequence Producer (#3333, @embg)
|
||||
api: Support for in-place decompression (#3432, @terrelln)
|
||||
api: New `ZSTD_CCtx_setCParams()` function, set all parameters defined in a `ZSTD_compressionParameters` structure (#3403, @Cyan4973)
|
||||
api: Streaming decompression detects incorrect header ID sooner (#3175, @Cyan4973)
|
||||
|
||||
4
Makefile
4
Makefile
@ -123,7 +123,7 @@ contrib: lib
|
||||
$(MAKE) -C contrib/seekable_format/examples all
|
||||
$(MAKE) -C contrib/seekable_format/tests test
|
||||
$(MAKE) -C contrib/largeNbDicts all
|
||||
$(MAKE) -C contrib/externalMatchfinder all
|
||||
$(MAKE) -C contrib/externalSequenceProducer all
|
||||
cd build/single_file_libs/ ; ./build_decoder_test.sh
|
||||
cd build/single_file_libs/ ; ./build_library_test.sh
|
||||
|
||||
@ -143,7 +143,7 @@ clean:
|
||||
$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/externalMatchfinder $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID)
|
||||
$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||
$(Q)$(RM) -r lz4
|
||||
@echo Cleaning completed
|
||||
|
||||
2
contrib/externalMatchfinder/.gitignore
vendored
2
contrib/externalMatchfinder/.gitignore
vendored
@ -1,2 +0,0 @@
|
||||
# build artifacts
|
||||
externalMatchfinder
|
||||
@ -1,14 +0,0 @@
|
||||
externalMatchfinder
|
||||
=====================
|
||||
|
||||
`externalMatchfinder` is a test tool for the external matchfinder API.
|
||||
It demonstrates how to use the API to perform a simple round-trip test.
|
||||
|
||||
A sample matchfinder is provided in matchfinder.c, but the user can swap
|
||||
this out with a different one if desired. The sample matchfinder implements
|
||||
LZ compression with a 1KB hashtable. Dictionary compression is not currently supported.
|
||||
|
||||
Command line :
|
||||
```
|
||||
externalMatchfinder filename
|
||||
```
|
||||
2
contrib/externalSequenceProducer/.gitignore
vendored
Normal file
2
contrib/externalSequenceProducer/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
# build artifacts
|
||||
externalSequenceProducer
|
||||
@ -23,11 +23,11 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wredundant-decls
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
|
||||
default: externalMatchfinder
|
||||
default: externalSequenceProducer
|
||||
|
||||
all: externalMatchfinder
|
||||
all: externalSequenceProducer
|
||||
|
||||
externalMatchfinder: matchfinder.c main.c $(LIBZSTD)
|
||||
externalSequenceProducer: sequence_producer.c main.c $(LIBZSTD)
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
.PHONY: $(LIBZSTD)
|
||||
@ -37,4 +37,4 @@ $(LIBZSTD):
|
||||
clean:
|
||||
$(RM) *.o
|
||||
$(MAKE) -C $(LIBDIR) clean > /dev/null
|
||||
$(RM) externalMatchfinder
|
||||
$(RM) externalSequenceProducer
|
||||
14
contrib/externalSequenceProducer/README.md
Normal file
14
contrib/externalSequenceProducer/README.md
Normal file
@ -0,0 +1,14 @@
|
||||
externalSequenceProducer
|
||||
=====================
|
||||
|
||||
`externalSequenceProducer` is a test tool for the Block-Level Sequence Producer API.
|
||||
It demonstrates how to use the API to perform a simple round-trip test.
|
||||
|
||||
A sample sequence producer is provided in sequence_producer.c, but the user can swap
|
||||
this out with a different one if desired. The sample sequence producer implements
|
||||
LZ parsing with a 1KB hashtable. Dictionary-based parsing is not currently supported.
|
||||
|
||||
Command line :
|
||||
```
|
||||
externalSequenceProducer filename
|
||||
```
|
||||
@ -16,7 +16,7 @@
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "zstd_errors.h"
|
||||
#include "matchfinder.h" // simpleExternalMatchFinder
|
||||
#include "sequence_producer.h" // simpleSequenceProducer
|
||||
|
||||
#define CHECK(res) \
|
||||
do { \
|
||||
@ -28,23 +28,23 @@ do { \
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
if (argc != 2) {
|
||||
printf("Usage: exampleMatchfinder <file>\n");
|
||||
printf("Usage: externalSequenceProducer <file>\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
ZSTD_CCtx* const zc = ZSTD_createCCtx();
|
||||
|
||||
int simpleExternalMatchState = 0xdeadbeef;
|
||||
int simpleSequenceProducerState = 0xdeadbeef;
|
||||
|
||||
// Here is the crucial bit of code!
|
||||
ZSTD_registerExternalMatchFinder(
|
||||
ZSTD_registerSequenceProducer(
|
||||
zc,
|
||||
&simpleExternalMatchState,
|
||||
simpleExternalMatchFinder
|
||||
&simpleSequenceProducerState,
|
||||
simpleSequenceProducer
|
||||
);
|
||||
|
||||
{
|
||||
size_t const res = ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 1);
|
||||
size_t const res = ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 1);
|
||||
CHECK(res);
|
||||
}
|
||||
|
||||
@ -9,15 +9,15 @@
|
||||
*/
|
||||
|
||||
#include "zstd_compress_internal.h"
|
||||
#include "matchfinder.h"
|
||||
#include "sequence_producer.h"
|
||||
|
||||
#define HSIZE 1024
|
||||
static U32 const HLOG = 10;
|
||||
static U32 const MLS = 4;
|
||||
static U32 const BADIDX = 0xffffffff;
|
||||
|
||||
size_t simpleExternalMatchFinder(
|
||||
void* externalMatchState,
|
||||
size_t simpleSequenceProducer(
|
||||
void* sequenceProducerState,
|
||||
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize,
|
||||
@ -31,7 +31,7 @@ size_t simpleExternalMatchFinder(
|
||||
size_t seqCount = 0;
|
||||
U32 hashTable[HSIZE];
|
||||
|
||||
(void)externalMatchState;
|
||||
(void)sequenceProducerState;
|
||||
(void)dict;
|
||||
(void)dictSize;
|
||||
(void)outSeqsCapacity;
|
||||
@ -14,8 +14,8 @@
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
|
||||
size_t simpleExternalMatchFinder(
|
||||
void* externalMatchState,
|
||||
size_t simpleSequenceProducer(
|
||||
void* sequenceProducerState,
|
||||
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize,
|
||||
@ -54,7 +54,7 @@ const char* ERR_getErrorString(ERR_enum code)
|
||||
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
|
||||
case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
|
||||
case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
|
||||
case PREFIX(externalMatchFinder_failed): return "External matchfinder returned an error code";
|
||||
case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code";
|
||||
case PREFIX(externalSequences_invalid): return "External sequences are not valid";
|
||||
case PREFIX(maxCode):
|
||||
default: return notErrorCode;
|
||||
|
||||
@ -615,7 +615,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
|
||||
bounds.upperBound = (int)ZSTD_ps_disable;
|
||||
return bounds;
|
||||
|
||||
case ZSTD_c_enableMatchFinderFallback:
|
||||
case ZSTD_c_enableSeqProducerFallback:
|
||||
bounds.lowerBound = 0;
|
||||
bounds.upperBound = 1;
|
||||
return bounds;
|
||||
@ -695,7 +695,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
|
||||
case ZSTD_c_useRowMatchFinder:
|
||||
case ZSTD_c_deterministicRefPrefix:
|
||||
case ZSTD_c_prefetchCDictTables:
|
||||
case ZSTD_c_enableMatchFinderFallback:
|
||||
case ZSTD_c_enableSeqProducerFallback:
|
||||
case ZSTD_c_maxBlockSize:
|
||||
case ZSTD_c_searchForExternalRepcodes:
|
||||
default:
|
||||
@ -754,7 +754,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
|
||||
case ZSTD_c_useRowMatchFinder:
|
||||
case ZSTD_c_deterministicRefPrefix:
|
||||
case ZSTD_c_prefetchCDictTables:
|
||||
case ZSTD_c_enableMatchFinderFallback:
|
||||
case ZSTD_c_enableSeqProducerFallback:
|
||||
case ZSTD_c_maxBlockSize:
|
||||
case ZSTD_c_searchForExternalRepcodes:
|
||||
break;
|
||||
@ -989,8 +989,8 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
|
||||
CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value;
|
||||
return CCtxParams->prefetchCDictTables;
|
||||
|
||||
case ZSTD_c_enableMatchFinderFallback:
|
||||
BOUNDCHECK(ZSTD_c_enableMatchFinderFallback, value);
|
||||
case ZSTD_c_enableSeqProducerFallback:
|
||||
BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value);
|
||||
CCtxParams->enableMatchFinderFallback = value;
|
||||
return CCtxParams->enableMatchFinderFallback;
|
||||
|
||||
@ -1140,7 +1140,7 @@ size_t ZSTD_CCtxParams_getParameter(
|
||||
case ZSTD_c_prefetchCDictTables:
|
||||
*value = (int)CCtxParams->prefetchCDictTables;
|
||||
break;
|
||||
case ZSTD_c_enableMatchFinderFallback:
|
||||
case ZSTD_c_enableSeqProducerFallback:
|
||||
*value = CCtxParams->enableMatchFinderFallback;
|
||||
break;
|
||||
case ZSTD_c_maxBlockSize:
|
||||
@ -1610,8 +1610,8 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
|
||||
|
||||
/* Helper function for calculating memory requirements.
|
||||
* Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */
|
||||
static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useExternalMatchFinder) {
|
||||
U32 const divider = (minMatch==3 || useExternalMatchFinder) ? 3 : 4;
|
||||
static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) {
|
||||
U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4;
|
||||
return blockSize / divider;
|
||||
}
|
||||
|
||||
@ -1623,12 +1623,12 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
|
||||
const size_t buffInSize,
|
||||
const size_t buffOutSize,
|
||||
const U64 pledgedSrcSize,
|
||||
int useExternalMatchFinder,
|
||||
int useSequenceProducer,
|
||||
size_t maxBlockSize)
|
||||
{
|
||||
size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
|
||||
size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
|
||||
size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useExternalMatchFinder);
|
||||
size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer);
|
||||
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
|
||||
+ ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
|
||||
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
|
||||
@ -1648,7 +1648,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
|
||||
size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
|
||||
|
||||
size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
|
||||
size_t const externalSeqSpace = useExternalMatchFinder
|
||||
size_t const externalSeqSpace = useSequenceProducer
|
||||
? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
|
||||
: 0;
|
||||
|
||||
@ -1679,7 +1679,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
|
||||
* be needed. However, we still allocate two 0-sized buffers, which can
|
||||
* take space under ASAN. */
|
||||
return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
|
||||
&cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
|
||||
&cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize);
|
||||
}
|
||||
|
||||
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
|
||||
@ -1740,7 +1740,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
|
||||
|
||||
return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
|
||||
&cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
|
||||
ZSTD_CONTENTSIZE_UNKNOWN, params->useExternalMatchFinder, params->maxBlockSize);
|
||||
ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2024,7 +2024,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
||||
|
||||
{ size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
|
||||
size_t const blockSize = MIN(params->maxBlockSize, windowSize);
|
||||
size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useExternalMatchFinder);
|
||||
size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useSequenceProducer);
|
||||
size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
|
||||
? ZSTD_compressBound(blockSize) + 1
|
||||
: 0;
|
||||
@ -2041,7 +2041,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
||||
size_t const neededSpace =
|
||||
ZSTD_estimateCCtxSize_usingCCtxParams_internal(
|
||||
¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
|
||||
buffInSize, buffOutSize, pledgedSrcSize, params->useExternalMatchFinder, params->maxBlockSize);
|
||||
buffInSize, buffOutSize, pledgedSrcSize, params->useSequenceProducer, params->maxBlockSize);
|
||||
int resizeWorkspace;
|
||||
|
||||
FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
|
||||
@ -2155,7 +2155,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
||||
}
|
||||
|
||||
/* reserve space for block-level external sequences */
|
||||
if (params->useExternalMatchFinder) {
|
||||
if (params->useSequenceProducer) {
|
||||
size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
|
||||
zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq;
|
||||
zc->externalMatchCtx.seqBuffer =
|
||||
@ -3022,26 +3022,26 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
|
||||
ssPtr->longLengthType = ZSTD_llt_none;
|
||||
}
|
||||
|
||||
/* ZSTD_postProcessExternalMatchFinderResult() :
|
||||
/* ZSTD_postProcessSequenceProducerResult() :
|
||||
* Validates and post-processes sequences obtained through the external matchfinder API:
|
||||
* - Checks whether nbExternalSeqs represents an error condition.
|
||||
* - Appends a block delimiter to outSeqs if one is not already present.
|
||||
* See zstd.h for context regarding block delimiters.
|
||||
* Returns the number of sequences after post-processing, or an error code. */
|
||||
static size_t ZSTD_postProcessExternalMatchFinderResult(
|
||||
static size_t ZSTD_postProcessSequenceProducerResult(
|
||||
ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize
|
||||
) {
|
||||
RETURN_ERROR_IF(
|
||||
nbExternalSeqs > outSeqsCapacity,
|
||||
externalMatchFinder_failed,
|
||||
"External matchfinder returned error code %lu",
|
||||
sequenceProducer_failed,
|
||||
"External sequence producer returned error code %lu",
|
||||
(unsigned long)nbExternalSeqs
|
||||
);
|
||||
|
||||
RETURN_ERROR_IF(
|
||||
nbExternalSeqs == 0 && srcSize > 0,
|
||||
externalMatchFinder_failed,
|
||||
"External matchfinder produced zero sequences for a non-empty src buffer!"
|
||||
sequenceProducer_failed,
|
||||
"Got zero sequences from external sequence producer for a non-empty src buffer!"
|
||||
);
|
||||
|
||||
if (srcSize == 0) {
|
||||
@ -3061,7 +3061,7 @@ static size_t ZSTD_postProcessExternalMatchFinderResult(
|
||||
* produced an invalid parse, by definition of ZSTD_sequenceBound(). */
|
||||
RETURN_ERROR_IF(
|
||||
nbExternalSeqs == outSeqsCapacity,
|
||||
externalMatchFinder_failed,
|
||||
sequenceProducer_failed,
|
||||
"nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!"
|
||||
);
|
||||
|
||||
@ -3139,9 +3139,9 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
|
||||
/* External matchfinder + LDM is technically possible, just not implemented yet.
|
||||
* We need to revisit soon and implement it. */
|
||||
RETURN_ERROR_IF(
|
||||
zc->appliedParams.useExternalMatchFinder,
|
||||
zc->appliedParams.useSequenceProducer,
|
||||
parameter_combination_unsupported,
|
||||
"Long-distance matching with external matchfinder enabled is not currently supported."
|
||||
"Long-distance matching with external sequence producer enabled is not currently supported."
|
||||
);
|
||||
|
||||
/* Updates ldmSeqStore.pos */
|
||||
@ -3158,9 +3158,9 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
|
||||
/* External matchfinder + LDM is technically possible, just not implemented yet.
|
||||
* We need to revisit soon and implement it. */
|
||||
RETURN_ERROR_IF(
|
||||
zc->appliedParams.useExternalMatchFinder,
|
||||
zc->appliedParams.useSequenceProducer,
|
||||
parameter_combination_unsupported,
|
||||
"Long-distance matching with external matchfinder enabled is not currently supported."
|
||||
"Long-distance matching with external sequence producer enabled is not currently supported."
|
||||
);
|
||||
|
||||
ldmSeqStore.seq = zc->ldmSequences;
|
||||
@ -3177,7 +3177,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
|
||||
zc->appliedParams.useRowMatchFinder,
|
||||
src, srcSize);
|
||||
assert(ldmSeqStore.pos == ldmSeqStore.size);
|
||||
} else if (zc->appliedParams.useExternalMatchFinder) {
|
||||
} else if (zc->appliedParams.useSequenceProducer) {
|
||||
assert(
|
||||
zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize)
|
||||
);
|
||||
@ -3195,7 +3195,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
|
||||
windowSize
|
||||
);
|
||||
|
||||
size_t const nbPostProcessedSeqs = ZSTD_postProcessExternalMatchFinderResult(
|
||||
size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult(
|
||||
zc->externalMatchCtx.seqBuffer,
|
||||
nbExternalSeqs,
|
||||
zc->externalMatchCtx.seqBufferCapacity,
|
||||
@ -3217,7 +3217,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
|
||||
"Failed to copy external sequences to seqStore!"
|
||||
);
|
||||
ms->ldmSeqStore = NULL;
|
||||
DEBUGLOG(5, "Copied %lu sequences from external matchfinder to internal seqStore.", (unsigned long)nbExternalSeqs);
|
||||
DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs);
|
||||
return ZSTDbss_compress;
|
||||
}
|
||||
|
||||
@ -3233,7 +3233,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
|
||||
ms->ldmSeqStore = NULL;
|
||||
DEBUGLOG(
|
||||
5,
|
||||
"External matchfinder returned error code %lu. Falling back to internal matchfinder.",
|
||||
"External sequence producer returned error code %lu. Falling back to internal parser.",
|
||||
(unsigned long)nbExternalSeqs
|
||||
);
|
||||
lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
|
||||
@ -6033,9 +6033,9 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
/* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */
|
||||
RETURN_ERROR_IF(
|
||||
params.useExternalMatchFinder == 1 && params.nbWorkers >= 1,
|
||||
params.useSequenceProducer == 1 && params.nbWorkers >= 1,
|
||||
parameter_combination_unsupported,
|
||||
"External matchfinder isn't supported with nbWorkers >= 1"
|
||||
"External sequence producer isn't supported with nbWorkers >= 1"
|
||||
);
|
||||
|
||||
if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
|
||||
@ -6251,7 +6251,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
|
||||
*/
|
||||
static size_t
|
||||
ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
|
||||
size_t posInSrc, U32 windowLog, size_t dictSize, int useExternalMatchFinder)
|
||||
size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer)
|
||||
{
|
||||
U32 const windowSize = 1u << windowLog;
|
||||
/* posInSrc represents the amount of data the decoder would decode up to this point.
|
||||
@ -6260,7 +6260,7 @@ ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
|
||||
* window size. After output surpasses windowSize, we're limited to windowSize offsets again.
|
||||
*/
|
||||
size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
|
||||
size_t const matchLenLowerBound = (minMatch == 3 || useExternalMatchFinder) ? 3 : 4;
|
||||
size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4;
|
||||
RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!");
|
||||
/* Validate maxNbSeq is large enough for the given matchLength and minMatch */
|
||||
RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch");
|
||||
@ -6325,7 +6325,7 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
|
||||
if (cctx->appliedParams.validateSequences) {
|
||||
seqPos->posInSrc += litLength + matchLength;
|
||||
FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
|
||||
cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useExternalMatchFinder),
|
||||
cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer),
|
||||
"Sequence validation failed");
|
||||
}
|
||||
RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
|
||||
@ -6463,7 +6463,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
|
||||
if (cctx->appliedParams.validateSequences) {
|
||||
seqPos->posInSrc += litLength + matchLength;
|
||||
FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
|
||||
cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useExternalMatchFinder),
|
||||
cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer),
|
||||
"Sequence validation failed");
|
||||
}
|
||||
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
|
||||
@ -6908,9 +6908,9 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
|
||||
return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
|
||||
}
|
||||
|
||||
void ZSTD_registerExternalMatchFinder(
|
||||
void ZSTD_registerSequenceProducer(
|
||||
ZSTD_CCtx* zc, void* mState,
|
||||
ZSTD_externalMatchFinder_F* mFinder
|
||||
ZSTD_sequenceProducer_F* mFinder
|
||||
) {
|
||||
if (mFinder != NULL) {
|
||||
ZSTD_externalMatchCtx emctx;
|
||||
@ -6919,9 +6919,9 @@ void ZSTD_registerExternalMatchFinder(
|
||||
emctx.seqBuffer = NULL;
|
||||
emctx.seqBufferCapacity = 0;
|
||||
zc->externalMatchCtx = emctx;
|
||||
zc->requestedParams.useExternalMatchFinder = 1;
|
||||
zc->requestedParams.useSequenceProducer = 1;
|
||||
} else {
|
||||
ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
|
||||
zc->requestedParams.useExternalMatchFinder = 0;
|
||||
zc->requestedParams.useSequenceProducer = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@ -353,8 +353,8 @@ struct ZSTD_CCtx_params_s {
|
||||
|
||||
/* Indicates whether an external matchfinder has been referenced.
|
||||
* Users can't set this externally.
|
||||
* It is set internally in ZSTD_registerExternalMatchFinder(). */
|
||||
int useExternalMatchFinder;
|
||||
* It is set internally in ZSTD_registerSequenceProducer(). */
|
||||
int useSequenceProducer;
|
||||
|
||||
/* Adjust the max block size*/
|
||||
size_t maxBlockSize;
|
||||
@ -395,7 +395,7 @@ typedef struct {
|
||||
/* Context for block-level external matchfinder API */
|
||||
typedef struct {
|
||||
void* mState;
|
||||
ZSTD_externalMatchFinder_F* mFinder;
|
||||
ZSTD_sequenceProducer_F* mFinder;
|
||||
ZSTD_Sequence* seqBuffer;
|
||||
size_t seqBufferCapacity;
|
||||
} ZSTD_externalMatchCtx;
|
||||
|
||||
201
lib/zstd.h
201
lib/zstd.h
@ -478,7 +478,7 @@ typedef enum {
|
||||
* ZSTD_c_useBlockSplitter
|
||||
* ZSTD_c_useRowMatchFinder
|
||||
* ZSTD_c_prefetchCDictTables
|
||||
* ZSTD_c_enableMatchFinderFallback
|
||||
* ZSTD_c_enableSeqProducerFallback
|
||||
* ZSTD_c_maxBlockSize
|
||||
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
|
||||
* note : never ever use experimentalParam? names directly;
|
||||
@ -565,7 +565,7 @@ typedef enum {
|
||||
* They will be used to compress next frame.
|
||||
* Resetting session never fails.
|
||||
* - The parameters : changes all parameters back to "default".
|
||||
* This also removes any reference to any dictionary or external matchfinder.
|
||||
* This also removes any reference to any dictionary or external sequence producer.
|
||||
* Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
|
||||
* otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
|
||||
* - Both : similar to resetting the session, followed by resetting parameters.
|
||||
@ -1627,8 +1627,8 @@ ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
|
||||
* Note : only single-threaded compression is supported.
|
||||
* ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
|
||||
*
|
||||
* Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the external matchfinder API at this time.
|
||||
* Size estimates assume that no external matchfinder is registered.
|
||||
* Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
|
||||
* Size estimates assume that no external sequence producer is registered.
|
||||
*/
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
|
||||
@ -1650,8 +1650,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
|
||||
* In this case, get total size by adding ZSTD_estimate?DictSize
|
||||
* Note 2 : only single-threaded compression is supported.
|
||||
* ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
|
||||
* Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the external matchfinder API at this time.
|
||||
* Size estimates assume that no external matchfinder is registered.
|
||||
* Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
|
||||
* Size estimates assume that no external sequence producer is registered.
|
||||
*/
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
|
||||
@ -2113,19 +2113,19 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
|
||||
*/
|
||||
#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
|
||||
|
||||
/* ZSTD_c_enableMatchFinderFallback
|
||||
/* ZSTD_c_enableSeqProducerFallback
|
||||
* Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
|
||||
*
|
||||
* Controls whether zstd will fall back to an internal matchfinder if an
|
||||
* external matchfinder is registered and returns an error code. This fallback is
|
||||
* block-by-block: the internal matchfinder will only be called for blocks where
|
||||
* the external matchfinder returns an error code. Fallback compression will
|
||||
* Controls whether zstd will fall back to an internal sequence producer if an
|
||||
* external sequence producer is registered and returns an error code. This fallback
|
||||
* is block-by-block: the internal sequence producer will only be called for blocks
|
||||
* where the external sequence producer returns an error code. Fallback parsing will
|
||||
* follow any other cParam settings, such as compression level, the same as in a
|
||||
* normal (fully-internal) compression operation.
|
||||
*
|
||||
* The user is strongly encouraged to read the full external matchfinder API
|
||||
* The user is strongly encouraged to read the full Block-Level Sequence Producer API
|
||||
* documentation (below) before setting this parameter. */
|
||||
#define ZSTD_c_enableMatchFinderFallback ZSTD_c_experimentalParam17
|
||||
#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
|
||||
|
||||
/* ZSTD_c_maxBlockSize
|
||||
* Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
|
||||
@ -2141,12 +2141,13 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
|
||||
|
||||
/* ZSTD_c_searchForExternalRepcodes
|
||||
* This parameter affects how zstd parses external sequences, such as sequences
|
||||
* provided through the compressSequences() API or from an external matchfinder.
|
||||
* provided through the compressSequences() API or from an external block-level
|
||||
* sequence producer.
|
||||
*
|
||||
* If set to ZSTD_ps_enable, the library will check for repeated offsets in
|
||||
* external sequences, even if those repcodes are not explicitly indicated in
|
||||
* the "rep" field. Note that this is the only way to exploit repcode matches
|
||||
* while using compressSequences() or an external matchfinder, since zstd
|
||||
* while using compressSequences() or an external sequence producer, since zstd
|
||||
* currently ignores the "rep" field of external sequences.
|
||||
*
|
||||
* If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
|
||||
@ -2805,43 +2806,52 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_
|
||||
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
|
||||
|
||||
|
||||
/* ********************** EXTERNAL MATCHFINDER API **********************
|
||||
/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
|
||||
*
|
||||
* *** OVERVIEW ***
|
||||
* This API allows users to replace the zstd internal block-level matchfinder
|
||||
* with an external matchfinder function. Potential applications of the API
|
||||
* include hardware-accelerated matchfinders and matchfinders specialized to
|
||||
* particular types of data.
|
||||
* The Block-Level Sequence Producer API allows users to provide their own custom
|
||||
* sequence producer which libzstd invokes to process each block. The produced list
|
||||
* of sequences (literals and matches) is then post-processed by libzstd to produce
|
||||
* valid compressed blocks.
|
||||
*
|
||||
* See contrib/externalMatchfinder for an example program employing the
|
||||
* external matchfinder API.
|
||||
* This block-level offload API is a more granular complement of the existing
|
||||
* frame-level offload API compressSequences() (introduced in v1.5.1). It offers
|
||||
* an easier migration story for applications already integrated with libzstd: the
|
||||
* user application continues to invoke the same compression functions
|
||||
* ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
|
||||
* from the specific advantages of the external sequence producer. For example,
|
||||
* the sequence producer could be tuned to take advantage of known characteristics
|
||||
* of the input, to offer better speed / ratio, or could leverage hardware
|
||||
* acceleration not available within libzstd itself.
|
||||
*
|
||||
* See contrib/externalSequenceProducer for an example program employing the
|
||||
* Block-Level Sequence Producer API.
|
||||
*
|
||||
* *** USAGE ***
|
||||
* The user is responsible for implementing a function of type
|
||||
* ZSTD_externalMatchFinder_F. For each block, zstd will pass the following
|
||||
* ZSTD_sequenceProducer_F. For each block, zstd will pass the following
|
||||
* arguments to the user-provided function:
|
||||
*
|
||||
* - externalMatchState: a pointer to a user-managed state for the external
|
||||
* matchfinder.
|
||||
* - sequenceProducerState: a pointer to a user-managed state for the sequence
|
||||
* producer.
|
||||
*
|
||||
* - outSeqs, outSeqsCapacity: an output buffer for sequences produced by the
|
||||
* external matchfinder. outSeqsCapacity is guaranteed >=
|
||||
* ZSTD_sequenceBound(srcSize). The memory backing outSeqs is managed by
|
||||
* the CCtx.
|
||||
* - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
|
||||
* outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
|
||||
* backing outSeqs is managed by the CCtx.
|
||||
*
|
||||
* - src, srcSize: an input buffer which the external matchfinder must parse
|
||||
* into sequences. srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
|
||||
* - src, srcSize: an input buffer for the sequence producer to parse.
|
||||
* srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
|
||||
*
|
||||
* - dict, dictSize: a history buffer, which may be empty, which the external
|
||||
* matchfinder may reference as it produces sequences for the src buffer.
|
||||
* Currently, zstd will always pass dictSize == 0 into external matchfinders,
|
||||
* but this will change in the future.
|
||||
* - dict, dictSize: a history buffer, which may be empty, which the sequence
|
||||
* producer may reference as it parses the src buffer. Currently, zstd will
|
||||
* always pass dictSize == 0 into external sequence producers, but this will
|
||||
* change in the future.
|
||||
*
|
||||
* - compressionLevel: a signed integer representing the zstd compression level
|
||||
* set by the user for the current operation. The external matchfinder may
|
||||
* choose to use this information to change its compression strategy and
|
||||
* speed/ratio tradeoff. Note: The compression level does not reflect zstd
|
||||
* parameters set through the advanced API.
|
||||
* set by the user for the current operation. The sequence producer may choose
|
||||
* to use this information to change its compression strategy and speed/ratio
|
||||
* tradeoff. Note: the compression level does not reflect zstd parameters set
|
||||
* through the advanced API.
|
||||
*
|
||||
* - windowSize: a size_t representing the maximum allowed offset for external
|
||||
* sequences. Note that sequence offsets are sometimes allowed to exceed the
|
||||
@ -2851,7 +2861,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* bloc
|
||||
* The user-provided function shall return a size_t representing the number of
|
||||
* sequences written to outSeqs. This return value will be treated as an error
|
||||
* code if it is greater than outSeqsCapacity. The return value must be non-zero
|
||||
* if srcSize is non-zero. The ZSTD_EXTERNAL_MATCHFINDER_ERROR macro is provided
|
||||
* if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
|
||||
* for convenience, but any value greater than outSeqsCapacity will be treated as
|
||||
* an error code.
|
||||
*
|
||||
@ -2859,68 +2869,71 @@ ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* bloc
|
||||
* written to outSeqs must be a valid parse of the src buffer. Data corruption may
|
||||
* occur if the parse is not valid. A parse is defined to be valid if the
|
||||
* following conditions hold:
|
||||
* - The sum of matchLengths and literalLengths is equal to srcSize.
|
||||
* - All sequences in the parse have matchLength != 0, except for the final
|
||||
* sequence. matchLength is not constrained for the final sequence.
|
||||
* - All offsets respect the windowSize parameter as specified in
|
||||
* - The sum of matchLengths and literalLengths must equal srcSize.
|
||||
* - All sequences in the parse, except for the final sequence, must have
|
||||
* matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
|
||||
* matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
|
||||
* - All offsets must respect the windowSize parameter as specified in
|
||||
* doc/zstd_compression_format.md.
|
||||
* - If the final sequence has matchLength == 0, it must also have offset == 0.
|
||||
*
|
||||
* zstd will only validate these conditions (and fail compression if they do not
|
||||
* hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
|
||||
* validation has a performance cost.
|
||||
*
|
||||
* If the user-provided function returns an error, zstd will either fall back
|
||||
* to an internal matchfinder or fail the compression operation. The user can
|
||||
* choose between the two behaviors by setting the
|
||||
* ZSTD_c_enableMatchFinderFallback cParam. Fallback compression will follow any
|
||||
* other cParam settings, such as compression level, the same as in a normal
|
||||
* compression operation.
|
||||
* to an internal sequence producer or fail the compression operation. The user can
|
||||
* choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
|
||||
* cParam. Fallback compression will follow any other cParam settings, such as
|
||||
* compression level, the same as in a normal compression operation.
|
||||
*
|
||||
* The user shall instruct zstd to use a particular ZSTD_externalMatchFinder_F
|
||||
* function by calling ZSTD_registerExternalMatchFinder(cctx, externalMatchState,
|
||||
* externalMatchFinder). This setting will persist until the next parameter reset
|
||||
* of the CCtx.
|
||||
* The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
|
||||
* function by calling
|
||||
* ZSTD_registerSequenceProducer(cctx,
|
||||
* sequenceProducerState,
|
||||
* sequenceProducer)
|
||||
* This setting will persist until the next parameter reset of the CCtx.
|
||||
*
|
||||
* The externalMatchState must be initialized by the user before calling
|
||||
* ZSTD_registerExternalMatchFinder. The user is responsible for destroying the
|
||||
* externalMatchState.
|
||||
* The sequenceProducerState must be initialized by the user before calling
|
||||
* ZSTD_registerSequenceProducer(). The user is responsible for destroying the
|
||||
* sequenceProducerState.
|
||||
*
|
||||
* *** LIMITATIONS ***
|
||||
* External matchfinders are compatible with all zstd compression APIs which respect
|
||||
* advanced parameters. However, there are three limitations:
|
||||
* This API is compatible with all zstd compression APIs which respect advanced parameters.
|
||||
* However, there are three limitations:
|
||||
*
|
||||
* First, the ZSTD_c_enableLongDistanceMatching cParam is not supported.
|
||||
* COMPRESSION WILL FAIL if it is enabled and the user tries to compress with an
|
||||
* external matchfinder.
|
||||
* - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in
|
||||
* some cases (see its documentation for details). Users must explicitly set
|
||||
* ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an
|
||||
* external matchfinder is registered.
|
||||
* First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
|
||||
* COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
|
||||
* external sequence producer.
|
||||
* - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
|
||||
* cases (see its documentation for details). Users must explicitly set
|
||||
* ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
|
||||
* sequence producer is registered.
|
||||
* - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
|
||||
* whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
|
||||
* check the docs on ZSTD_c_enableLongDistanceMatching whenever the external
|
||||
* matchfinder API is used in conjunction with advanced settings (like windowLog).
|
||||
* check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
|
||||
* Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
|
||||
*
|
||||
* Second, history buffers are not supported. Concretely, zstd will always pass
|
||||
* dictSize == 0 to the external matchfinder (for now). This has two implications:
|
||||
* - Dictionaries are not supported. Compression will *not* fail if the user
|
||||
* Second, history buffers are not currently supported. Concretely, zstd will always pass
|
||||
* dictSize == 0 to the external sequence producer (for now). This has two implications:
|
||||
* - Dictionaries are not currently supported. Compression will *not* fail if the user
|
||||
* references a dictionary, but the dictionary won't have any effect.
|
||||
* - Stream history is not supported. All compression APIs, including streaming
|
||||
* APIs, work with the external matchfinder, but the external matchfinder won't
|
||||
* receive any history from the previous block. Each block is an independent chunk.
|
||||
* - Stream history is not currently supported. All advanced compression APIs, including
|
||||
* streaming APIs, work with external sequence producers, but each block is treated as
|
||||
* an independent chunk without history from previous blocks.
|
||||
*
|
||||
* Third, multi-threading within a single compression is not supported. In other words,
|
||||
* COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external matchfinder is registered.
|
||||
* Third, multi-threading within a single compression is not currently supported. In other words,
|
||||
* COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
|
||||
* Multi-threading across compressions is fine: simply create one CCtx per thread.
|
||||
*
|
||||
* Long-term, we plan to overcome all three limitations. There is no technical blocker to
|
||||
* overcoming them. It is purely a question of engineering effort.
|
||||
*/
|
||||
|
||||
#define ZSTD_EXTERNAL_MATCHFINDER_ERROR ((size_t)(-1))
|
||||
#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
|
||||
|
||||
typedef size_t ZSTD_externalMatchFinder_F (
|
||||
void* externalMatchState,
|
||||
typedef size_t ZSTD_sequenceProducer_F (
|
||||
void* sequenceProducerState,
|
||||
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize,
|
||||
@ -2928,32 +2941,30 @@ typedef size_t ZSTD_externalMatchFinder_F (
|
||||
size_t windowSize
|
||||
);
|
||||
|
||||
/*! ZSTD_registerExternalMatchFinder() :
|
||||
* Instruct zstd to use an external matchfinder function.
|
||||
/*! ZSTD_registerSequenceProducer() :
|
||||
* Instruct zstd to use a block-level external sequence producer function.
|
||||
*
|
||||
* The externalMatchState must be initialized by the caller, and the caller is
|
||||
* The sequenceProducerState must be initialized by the caller, and the caller is
|
||||
* responsible for managing its lifetime. This parameter is sticky across
|
||||
* compressions. It will remain set until the user explicitly resets compression
|
||||
* parameters.
|
||||
*
|
||||
* External matchfinder registration is considered to be an "advanced parameter",
|
||||
* part of the "advanced API". This means it will only have an effect on
|
||||
* compression APIs which respect advanced parameters, such as compress2() and
|
||||
* compressStream(). Older compression APIs such as compressCCtx(), which predate
|
||||
* the introduction of "advanced parameters", will ignore any external matchfinder
|
||||
* setting.
|
||||
* Sequence producer registration is considered to be an "advanced parameter",
|
||||
* part of the "advanced API". This means it will only have an effect on compression
|
||||
* APIs which respect advanced parameters, such as compress2() and compressStream2().
|
||||
* Older compression APIs such as compressCCtx(), which predate the introduction of
|
||||
* "advanced parameters", will ignore any external sequence producer setting.
|
||||
*
|
||||
* The external matchfinder can be "cleared" by registering a NULL external
|
||||
* matchfinder function pointer. This removes all limitations described above in
|
||||
* the "LIMITATIONS" section of the API docs.
|
||||
* The sequence producer can be "cleared" by registering a NULL function pointer. This
|
||||
* removes all limitations described above in the "LIMITATIONS" section of the API docs.
|
||||
*
|
||||
* The user is strongly encouraged to read the full API documentation (above)
|
||||
* before calling this function. */
|
||||
* The user is strongly encouraged to read the full API documentation (above) before
|
||||
* calling this function. */
|
||||
ZSTDLIB_STATIC_API void
|
||||
ZSTD_registerExternalMatchFinder(
|
||||
ZSTD_registerSequenceProducer(
|
||||
ZSTD_CCtx* cctx,
|
||||
void* externalMatchState,
|
||||
ZSTD_externalMatchFinder_F* externalMatchFinder
|
||||
void* sequenceProducerState,
|
||||
ZSTD_sequenceProducer_F* sequenceProducer
|
||||
);
|
||||
|
||||
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
|
||||
|
||||
@ -95,7 +95,7 @@ typedef enum {
|
||||
ZSTD_error_seekableIO = 102,
|
||||
ZSTD_error_dstBuffer_wrong = 104,
|
||||
ZSTD_error_srcBuffer_wrong = 105,
|
||||
ZSTD_error_externalMatchFinder_failed = 106,
|
||||
ZSTD_error_sequenceProducer_failed = 106,
|
||||
ZSTD_error_externalSequences_invalid = 107,
|
||||
ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
|
||||
} ZSTD_ErrorCode;
|
||||
|
||||
@ -17,8 +17,8 @@ static U32 const HLOG = 10;
|
||||
static U32 const MLS = 4;
|
||||
static U32 const BADIDX = 0xffffffff;
|
||||
|
||||
static size_t simpleExternalMatchFinder(
|
||||
void* externalMatchState,
|
||||
static size_t simpleSequenceProducer(
|
||||
void* sequenceProducerState,
|
||||
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize,
|
||||
@ -32,7 +32,7 @@ static size_t simpleExternalMatchFinder(
|
||||
size_t seqCount = 0;
|
||||
U32 hashTable[HSIZE];
|
||||
|
||||
(void)externalMatchState;
|
||||
(void)sequenceProducerState;
|
||||
(void)dict;
|
||||
(void)dictSize;
|
||||
(void)outSeqsCapacity;
|
||||
@ -80,15 +80,15 @@ static size_t simpleExternalMatchFinder(
|
||||
return seqCount;
|
||||
}
|
||||
|
||||
size_t zstreamExternalMatchFinder(
|
||||
void* externalMatchState,
|
||||
size_t zstreamSequenceProducer(
|
||||
void* sequenceProducerState,
|
||||
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize,
|
||||
int compressionLevel,
|
||||
size_t windowSize
|
||||
) {
|
||||
EMF_testCase const testCase = *((EMF_testCase*)externalMatchState);
|
||||
EMF_testCase const testCase = *((EMF_testCase*)sequenceProducerState);
|
||||
memset(outSeqs, 0, outSeqsCapacity);
|
||||
|
||||
switch (testCase) {
|
||||
@ -100,8 +100,8 @@ size_t zstreamExternalMatchFinder(
|
||||
outSeqs[0].litLength = (U32)(srcSize);
|
||||
return 1;
|
||||
case EMF_LOTS_OF_SEQS:
|
||||
return simpleExternalMatchFinder(
|
||||
externalMatchState,
|
||||
return simpleSequenceProducer(
|
||||
sequenceProducerState,
|
||||
outSeqs, outSeqsCapacity,
|
||||
src, srcSize,
|
||||
dict, dictSize,
|
||||
@ -135,6 +135,6 @@ size_t zstreamExternalMatchFinder(
|
||||
return outSeqsCapacity + 1;
|
||||
case EMF_BIG_ERROR:
|
||||
default:
|
||||
return ZSTD_EXTERNAL_MATCHFINDER_ERROR;
|
||||
return ZSTD_SEQUENCE_PRODUCER_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
@ -27,8 +27,8 @@ typedef enum {
|
||||
EMF_INVALID_LAST_LITS = 8
|
||||
} EMF_testCase;
|
||||
|
||||
size_t zstreamExternalMatchFinder(
|
||||
void* externalMatchState,
|
||||
size_t zstreamSequenceProducer(
|
||||
void* sequenceProducerState,
|
||||
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize,
|
||||
|
||||
@ -35,8 +35,8 @@ PRGDIR = ../../programs
|
||||
CONTRIBDIR = ../../contrib
|
||||
|
||||
# TODO(embg) make it possible to plug in an arbitrary matchfinder as a .o file
|
||||
MATCHFINDER_DIR = $(CONTRIBDIR)/externalMatchfinder
|
||||
MATCHFINDER_SRC = $(MATCHFINDER_DIR)/matchfinder.c
|
||||
MATCHFINDER_DIR = $(CONTRIBDIR)/externalSequenceProducer
|
||||
MATCHFINDER_SRC = $(MATCHFINDER_DIR)/sequence_producer.c
|
||||
|
||||
FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
||||
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
#include "fuzz_helpers.h"
|
||||
#include "zstd.h"
|
||||
#include "zdict.h"
|
||||
#include "matchfinder.h"
|
||||
#include "sequence_producer.h"
|
||||
|
||||
const int kMinClevel = -3;
|
||||
const int kMaxClevel = 19;
|
||||
@ -71,13 +71,13 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer)
|
||||
return params;
|
||||
}
|
||||
|
||||
static void setExternalMatchFinderParams(ZSTD_CCtx *cctx, FUZZ_dataProducer_t *producer) {
|
||||
ZSTD_registerExternalMatchFinder(
|
||||
static void setSequenceProducerParams(ZSTD_CCtx *cctx, FUZZ_dataProducer_t *producer) {
|
||||
ZSTD_registerSequenceProducer(
|
||||
cctx,
|
||||
NULL,
|
||||
simpleExternalMatchFinder
|
||||
simpleSequenceProducer
|
||||
);
|
||||
setRand(cctx, ZSTD_c_enableMatchFinderFallback, 0, 1, producer);
|
||||
setRand(cctx, ZSTD_c_enableSeqProducerFallback, 0, 1, producer);
|
||||
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0));
|
||||
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
|
||||
}
|
||||
@ -138,9 +138,9 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
|
||||
}
|
||||
|
||||
if (FUZZ_dataProducer_uint32Range(producer, 0, 10) == 1) {
|
||||
setExternalMatchFinderParams(cctx, producer);
|
||||
setSequenceProducerParams(cctx, producer);
|
||||
} else {
|
||||
ZSTD_registerExternalMatchFinder(cctx, NULL, NULL);
|
||||
ZSTD_registerSequenceProducer(cctx, NULL, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -40,7 +40,7 @@
|
||||
#include "seqgen.h"
|
||||
#include "util.h"
|
||||
#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
|
||||
#include "external_matchfinder.h" /* zstreamExternalMatchFinder, EMF_testCase */
|
||||
#include "external_matchfinder.h" /* zstreamSequenceProducer, EMF_testCase */
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
@ -1856,14 +1856,14 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
|
||||
}
|
||||
DISPLAYLEVEL(3, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "test%3i : External matchfinder API: ", testNb++);
|
||||
DISPLAYLEVEL(3, "test%3i : Block-Level External Sequence Producer API: ", testNb++);
|
||||
{
|
||||
size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
|
||||
BYTE* const dstBuf = (BYTE*)malloc(ZSTD_compressBound(dstBufSize));
|
||||
size_t const checkBufSize = CNBufferSize;
|
||||
BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
|
||||
int enableFallback;
|
||||
EMF_testCase externalMatchState;
|
||||
EMF_testCase sequenceProducerState;
|
||||
|
||||
CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
|
||||
|
||||
@ -1871,7 +1871,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
|
||||
|
||||
/* Reference external matchfinder outside the test loop to
|
||||
* check that the reference is preserved across compressions */
|
||||
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
|
||||
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
|
||||
|
||||
for (enableFallback = 0; enableFallback <= 1; enableFallback++) {
|
||||
size_t testCaseId;
|
||||
@ -1892,9 +1892,9 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
|
||||
ZSTD_ErrorCode const errorCodes[] = {
|
||||
ZSTD_error_no_error,
|
||||
ZSTD_error_no_error,
|
||||
ZSTD_error_externalMatchFinder_failed,
|
||||
ZSTD_error_externalMatchFinder_failed,
|
||||
ZSTD_error_externalMatchFinder_failed,
|
||||
ZSTD_error_sequenceProducer_failed,
|
||||
ZSTD_error_sequenceProducer_failed,
|
||||
ZSTD_error_sequenceProducer_failed,
|
||||
ZSTD_error_externalSequences_invalid,
|
||||
ZSTD_error_externalSequences_invalid,
|
||||
ZSTD_error_externalSequences_invalid,
|
||||
@ -1906,18 +1906,18 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
|
||||
|
||||
int const compressionShouldSucceed = (
|
||||
(errorCodes[testCaseId] == ZSTD_error_no_error) ||
|
||||
(enableFallback && errorCodes[testCaseId] == ZSTD_error_externalMatchFinder_failed)
|
||||
(enableFallback && errorCodes[testCaseId] == ZSTD_error_sequenceProducer_failed)
|
||||
);
|
||||
|
||||
int const testWithSequenceValidation = (
|
||||
testCases[testCaseId] == EMF_INVALID_OFFSET
|
||||
);
|
||||
|
||||
externalMatchState = testCases[testCaseId];
|
||||
sequenceProducerState = testCases[testCaseId];
|
||||
|
||||
ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, testWithSequenceValidation));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
|
||||
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
|
||||
|
||||
if (compressionShouldSucceed) {
|
||||
@ -1936,9 +1936,9 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
|
||||
/* Test compression with external matchfinder + empty src buffer */
|
||||
{
|
||||
size_t res;
|
||||
externalMatchState = EMF_ZERO_SEQS;
|
||||
sequenceProducerState = EMF_ZERO_SEQS;
|
||||
ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, enableFallback));
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
|
||||
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
|
||||
CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
|
||||
CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
|
||||
@ -1947,30 +1947,30 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
|
||||
|
||||
/* Test that reset clears the external matchfinder */
|
||||
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
|
||||
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
|
||||
sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
|
||||
CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
|
||||
|
||||
/* Test that registering mFinder == NULL clears the external matchfinder */
|
||||
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
|
||||
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
|
||||
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
|
||||
ZSTD_registerExternalMatchFinder(zc, NULL, NULL); /* clear the external matchfinder */
|
||||
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
|
||||
sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
|
||||
ZSTD_registerSequenceProducer(zc, NULL, NULL); /* clear the external matchfinder */
|
||||
CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
|
||||
|
||||
/* Test that external matchfinder doesn't interact with older APIs */
|
||||
ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
|
||||
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
|
||||
externalMatchState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 0));
|
||||
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
|
||||
sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
|
||||
CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
|
||||
|
||||
/* Test that compression returns the correct error with LDM */
|
||||
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
|
||||
{
|
||||
size_t res;
|
||||
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
|
||||
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
|
||||
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
|
||||
CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
|
||||
@ -1985,7 +1985,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
|
||||
CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
|
||||
{
|
||||
size_t res;
|
||||
ZSTD_registerExternalMatchFinder(zc, &externalMatchState, zstreamExternalMatchFinder);
|
||||
ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
|
||||
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
|
||||
res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
|
||||
CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
|
||||
|
||||
Reference in New Issue
Block a user