lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-08-05 19:15:58 +03:00
Code Activity

context can be sized down even with constant parameters

Browse Source 
when parameters are "equivalent",
the context is re-used in continue mode,
hence needed workspace size is not recalculated.
This incidentally also evades the size-down check and action.

This patch intercepts the "continue mode"
so that the size-down check and action is actually triggered.
This commit is contained in:
Yann Collet
2018-06-06 15:04:12 -07:00
parent e5e17d009f
commit f1ea383f45
2 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/compress/zstd_compress.c
View File

@@ -1054,10 +1054,10 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
return ptr; return ptr;
} }
#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as ZSTD_WORKSPACETOOLARGE_FACTOR times larger than needed */ #define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */
#define ZSTD_WORKSPACETOOLARGE_MAX 128 /* when workspace is continuously too large #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large
* at least that number of times, * during at least this number of times,
* context's memory usage is actually wasteful, * context's memory usage is considered wasteful,
* because it's sized to handle a worst case scenario which rarely happens. * because it's sized to handle a worst case scenario which rarely happens.
* In which case, resize it down to free some memory */ * In which case, resize it down to free some memory */
@@ -1080,7 +1080,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)", DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)",
zc->appliedParams.cParams.windowLog, zc->blockSize); zc->appliedParams.cParams.windowLog, zc->blockSize);
zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */ zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */
return ZSTD_continueCCtx(zc, params, pledgedSrcSize); if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION)
return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
} } } }
DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");
@@ -1117,7 +1118,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
int const workSpaceTooSmall = zc->workSpaceSize < neededSpace; int const workSpaceTooSmall = zc->workSpaceSize < neededSpace;
int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace;
int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAX); int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION);
zc->workSpaceOversizedDuration = workSpaceTooLarge ? zc->workSpaceOversizedDuration+1 : 0; zc->workSpaceOversizedDuration = workSpaceTooLarge ? zc->workSpaceOversizedDuration+1 : 0;
DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers",

3
tests/fuzzer.c
View File

@@ -490,8 +490,7 @@ static int basicUnitTests(U32 seed, double compressibility)
* make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */ * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
U32 u; U32 u;
for (u=0; u<maxNbAttempts; u++) { for (u=0; u<maxNbAttempts; u++) {
size_t const srcSize = (FUZ_rand(&seed) & 4095) + 200; CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize, -9));
if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */ if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
} }
DISPLAYLEVEL(5, "size down after %u attempts : ", u); DISPLAYLEVEL(5, "size down after %u attempts : ", u);