From eb21b7f48224b35ceb91b31a2fe1bb764545724e Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 9 Jul 2018 13:44:01 -0700 Subject: [PATCH] Not crashing --- programs/bench.c | 10 ++++---- tests/paramgrill.c | 59 +++++++++++++++++++++++++++------------------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 0d35f3eb3..5d5b47aba 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -288,7 +288,7 @@ volatile char g_touched; /* takes # of blocks and list of size & stuff for each. */ /* only does looping */ /* note time per loop could be zero if interval too short */ -BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( +BMK_customReturn_t BMK_benchFunction( BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, @@ -310,6 +310,7 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } { + unsigned i, j; for(i = 0; i < blockCount; i++) { for(j = 0; j < srcBlockSizes[i]; j++) { @@ -318,11 +319,11 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } for(i = 0; i < blockCount; i++) { memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ - //this is written at end proc? where did compressed data get overwritten ny this? } - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); + //UTIL_sleepMilli(5); /* give processor time to other processes */ + //UTIL_waitForNextTick(); + } { @@ -463,7 +464,6 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } { size_t const decodedSize = (size_t)totalDSize64; free(*resultBufferPtr); - //TODO: decodedSize corrupted? *resultBufferPtr = malloc(decodedSize); if (!(*resultBufferPtr)) { EXM_THROW(33, BMK_return_t, "not enough memory"); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 44bcedef8..77e9f2250 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -598,8 +598,8 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); //Make sure memory is at least close to feasible? //ZSTD_estimateCCtxSize thing. @@ -669,23 +669,27 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyPara //initializing memoTable /* */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen) { +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int wFixed = !paramConstraints.windowLog; int j = 0; memset(memoTable, 0, arrayLen); - + cParamZeroMin(¶mConstraints); for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - BMK_translateAdvancedParams(paramConstraints); if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { //infeasible; memoTable[i] = 255; j++; } + //TODO: remove any where memoTable wlog is mark any where windowlog is too big for data. + if(wFixed && (1 << paramConstraints.windowLog) > (srcSize << 1)) { + memoTable[i] = 255; + } /* nil out parameter sets equivalent to others. */ if(cwFixed/* at most least 1 param fixed. */) { if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { @@ -771,8 +775,8 @@ static ZSTD_compressionParameters randomParams(void) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); } return p; } @@ -1201,7 +1205,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1230,7 +1234,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1275,14 +1279,12 @@ static int infeasibleBench(BMK_result_t* resultPtr, BMK_advancedParams_t adv = BMK_initAdvancedParams(); BMK_return_t benchres; BMK_result_t resultMin, resultMax; - UTIL_time_t startTime; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS = resultScore(*winnerResult, srcSize, target); adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) - (void)startTime; //TODO: actually use this to adjust timing DISPLAY("WinnerScore: %f\n ", winnerRS); /* adv.loopMode = BMK_timeMode; @@ -1290,6 +1292,11 @@ static int infeasibleBench(BMK_result_t* resultPtr, benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + adv.loopMode = BMK_timeMode; + adv.nbSeconds = 1; + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + if(!benchres.error) { *resultPtr = benchres.result; if(eqZero(benchres.result.cSpeed)) { @@ -1372,6 +1379,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //probably pick a different code for already tested? @@ -1397,6 +1405,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, BMK_result_t* winnerResult, U8* memoTable, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns @@ -1422,7 +1431,6 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t // *actually if it performs too //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) -//TODO: paramTarget uneeded at this point w/ varArray and init; static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. @@ -1470,12 +1478,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); candidateInfo.params = cparam; //all dist-1 targets + //if we early end this, we should also randomize the order these are picked. for(i = 0; i < varLen; i++) { paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1498,8 +1507,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1587,8 +1596,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1605,8 +1614,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1681,9 +1690,11 @@ static winnerInfo_t optimizeFixedStrategy( /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - cParamZeroMin(¶mTarget); - memoTableInit(memoTable, paramTarget, target, varNew, varLenNew); + memoTableInit(memoTable, paramTarget, target, varNew, varLenNew, srcSize); + + //needs to happen after memoTableInit as that assumes 0 = undefined. + cParamZeroMin(¶mTarget); init = paramTarget; @@ -1699,7 +1710,7 @@ static winnerInfo_t optimizeFixedStrategy( candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; - DISPLAY("Climb Winner: "); + DISPLAY("New Winner: "); BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); } i++; @@ -1714,7 +1725,7 @@ _cleanUp: } // bigger and (hopefully) better* than optimizeForSize -// TODO: Change level bm'ing to respect constraints. +// TODO: allow accept multiple files like benchFiles or bench.c fn's static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" );