diff --git a/lib/fse.c b/lib/fse.c index 8b1223dad..4fc692f24 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -914,8 +914,8 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t else { bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } + ip = iend - 4; + } bitStream = FSE_readLE32(ip) >> (bitCount & 31); } } @@ -967,20 +967,20 @@ void FSE_freeCTable (FSE_CTable* ct) /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) { - U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1; - U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2; - U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; - return minBits; + U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + return minBits; } unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) { - U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2; + U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2; U32 tableLog = maxTableLog; - U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ - if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; return tableLog; @@ -1569,8 +1569,8 @@ size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= unsigned FSE_reloadDStream(FSE_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ - return FSE_DStream_tooFar; + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + return FSE_DStream_tooFar; if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { @@ -1834,7 +1834,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32 if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC; /* not implemented (not possible with current format) */ if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* not enough space within dst buffer */ op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1)); - huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */ + huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */ for (n=0; n= 0; pos--) { if (huffNode[pos].nbBits >= currentNbBits) continue; @@ -1917,20 +1917,20 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) } } - while (totalCost < 0) /* Sometimes, cost correction overshoot */ - { - if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */ - { - while (huffNode[n].nbBits == maxNbBits) n--; - huffNode[n+1].nbBits--; - rankLast[1] = n+1; - totalCost++; - continue; - } - huffNode[ rankLast[1] + 1 ].nbBits--; - rankLast[1]++; - totalCost ++; - } + while (totalCost < 0) /* Sometimes, cost correction overshoot */ + { + if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */ + { + while (huffNode[n].nbBits == maxNbBits) n--; + huffNode[n+1].nbBits--; + rankLast[1] = n+1; + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } } } @@ -1981,7 +1981,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3 /* safety checks */ if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG; if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC; - memset(huffNode0, 0, sizeof(huffNode0)); + memset(huffNode0, 0, sizeof(huffNode0)); // sort, decreasing order HUF_sort(huffNode, count, maxSymbolValue); @@ -2066,7 +2066,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size FSE_CStream_t bitC; /* init */ - if (dstSize < 8) return 0; + if (dstSize < 8) return 0; op += 6; /* jump Table -- could be optimized by delta / deviation */ errorCode = FSE_initCStream(&bitC, op, oend-op); if (FSE_isError(errorCode)) return 0; diff --git a/lib/zstd.c b/lib/zstd.c index 23fdff73b..a574efc9e 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -542,11 +542,11 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize, const size_t minGain = ZSTD_minGain(srcSize); BYTE* const ostart = (BYTE*)dst; size_t hsize; - static const size_t LHSIZE = 5; + static const size_t LHSIZE = 5; - if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space for compression */ + if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space for compression */ - hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize); + hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize); if (hsize<2) return hsize; /* special cases */ if (hsize >= srcSize - minGain) return 0; @@ -619,8 +619,8 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, } /* Sequences Header */ - if ((oend-op) < 2+3+6) /* nbSeq + dumpsLength + 3*rleCTable*/ - return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; + if ((oend-op) < 2+3+6) /* nbSeq + dumpsLength + 3*rleCTable*/ + return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; ZSTD_writeLE16(op, (U16)nbSeq); op+=2; seqHead = op; @@ -640,7 +640,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, op[2] = (BYTE)(dumpsLength); op += 3; } - if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; + if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; memcpy(op, seqStorePtr->dumpsStart, dumpsLength); op += dumpsLength; } @@ -661,11 +661,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, } else { - size_t NCountSize; + size_t NCountSize; tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); FSE_normalizeCount(norm, tableLog, count, nbSeq, max); - NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC; + NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC; op += NCountSize; FSE_buildCTable(CTable_LitLength, norm, max, tableLog); LLtype = bt_compressed; @@ -696,11 +696,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, } else { - size_t NCountSize; + size_t NCountSize; tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); FSE_normalizeCount(norm, tableLog, count, nbSeq, max); - NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC; + NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC; op += NCountSize; FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog); Offtype = bt_compressed; @@ -722,11 +722,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, } else { - size_t NCountSize; + size_t NCountSize; tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); FSE_normalizeCount(norm, tableLog, count, nbSeq, max); - NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC; + NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC; op += NCountSize; FSE_buildCTable(CTable_MatchLength, norm, max, tableLog); MLtype = bt_compressed; @@ -1028,7 +1028,7 @@ size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, con size_t blockSize = BLOCKSIZE; if (blockSize > srcSize) blockSize = srcSize; - if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */ + if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */ return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* update hash table */ @@ -1250,8 +1250,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, U32 LLlog, Offlog, MLlog; size_t dumpsLength; - /* check */ - if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize; + /* check */ + if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize; /* SeqHead */ *nbSeq = ZSTD_readLE16(ip); ip+=2; @@ -1273,8 +1273,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, *dumpsPtr = ip; ip += dumpsLength; - /* check */ - if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ + /* check */ + if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ /* sequences */ { @@ -1295,7 +1295,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, max = MaxLL; headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; - if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption; + if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption; ip += headerSize; FSE_buildDTable(DTableLL, norm, max, LLlog); } @@ -1314,7 +1314,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, max = MaxOff; headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; - if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption; + if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption; ip += headerSize; FSE_buildDTable(DTableOffb, norm, max, Offlog); } @@ -1333,7 +1333,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, max = MaxML; headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; - if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption; + if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption; ip += headerSize; FSE_buildDTable(DTableML, norm, max, MLlog); } @@ -1417,9 +1417,9 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) static size_t ZSTD_execSequence(BYTE* op, - seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - BYTE* const base, BYTE* const oend) + seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + BYTE* const base, BYTE* const oend) { static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */ @@ -1430,7 +1430,7 @@ static size_t ZSTD_execSequence(BYTE* op, /* check */ if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite beyond dst buffer */ - if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption; + if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption; if (sequence.matchLength > (size_t)(*litPtr-op)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite literal segment */ /* copy Literals */ @@ -1442,18 +1442,18 @@ static size_t ZSTD_execSequence(BYTE* op, *litPtr = litEnd; /* update for next sequence */ /* check : last match must be at a minimum distance of 8 from end of dest buffer */ - if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; + if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; - /* copy Match */ + /* copy Match */ { const U32 overlapRisk = (((size_t)(litEnd - endMatch)) < 12); const BYTE* match = op - sequence.offset; /* possible underflow at op - offset ? */ size_t qutt = 12; U64 saved[2]; - /* check */ - if (match < base) return (size_t)-ZSTD_ERROR_corruption; - if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption; + /* check */ + if (match < base) return (size_t)-ZSTD_ERROR_corruption; + if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption; /* save beginning of literal sequence, in case of write overlap */ if (overlapRisk) @@ -1473,7 +1473,7 @@ static size_t ZSTD_execSequence(BYTE* op, ZSTD_copy4(op+4, match); match -= dec64; } else { ZSTD_copy8(op, match); } - op += 8; match += 8; + op += 8; match += 8; if (endMatch > oend-12) { @@ -1498,11 +1498,11 @@ static size_t ZSTD_execSequence(BYTE* op, typedef struct ZSTD_Dctx_s { U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; - U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; - U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; + U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; + U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; void* previousDstEnd; void* base; - size_t expected; + size_t expected; blockType_t bType; U32 phase; } dctx_t; @@ -1514,7 +1514,7 @@ static size_t ZSTD_decompressSequences( const void* seqStart, size_t seqSize, const BYTE* litStart, size_t litSize) { - dctx_t* dctx = (dctx_t*)ctx; + dctx_t* dctx = (dctx_t*)ctx; const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; BYTE* const ostart = (BYTE* const)dst; @@ -1526,9 +1526,9 @@ static size_t ZSTD_decompressSequences( int nbSeq; const BYTE* dumps; U32* DTableLL = dctx->LLTable; - U32* DTableML = dctx->MLTable; + U32* DTableML = dctx->MLTable; U32* DTableOffb = dctx->OffTable; - BYTE* const base = (BYTE*) (dctx->base); + BYTE* const base = (BYTE*) (dctx->base); /* Build Decoding Tables */ errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, @@ -1658,8 +1658,8 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) { - dctx_t ctx; - ctx.base = dst; + dctx_t ctx; + ctx.base = dst; return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } @@ -1668,14 +1668,20 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src * Streaming Decompression API *******************************/ +size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx) +{ + dctx->expected = ZSTD_frameHeaderSize; + dctx->phase = 0; + dctx->previousDstEnd = NULL; + dctx->base = NULL; + return 0; +} + ZSTD_Dctx* ZSTD_createDCtx(void) { ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx)); if (dctx==NULL) return NULL; - dctx->expected = ZSTD_frameHeaderSize; - dctx->phase = 0; - dctx->previousDstEnd = NULL; - dctx->base = NULL; + ZSTD_resetDCtx(dctx); return dctx; } @@ -1685,7 +1691,6 @@ size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx) return 0; } - size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx) { return ((dctx_t*)dctx)->expected; @@ -1697,8 +1702,8 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co /* Sanity check */ if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize; - if (dst != ctx->previousDstEnd) /* not contiguous */ - ctx->base = dst; + if (dst != ctx->previousDstEnd) /* not contiguous */ + ctx->base = dst; /* Decompress : frame header */ if (ctx->phase == 0) @@ -1754,7 +1759,7 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co } ctx->phase = 1; ctx->expected = ZSTD_blockHeaderSize; - ctx->previousDstEnd = (void*)( ((char*)dst) + rSize); + ctx->previousDstEnd = (void*)( ((char*)dst) + rSize); return rSize; } diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 1baa47d32..55a41012f 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -56,6 +56,7 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize); typedef struct ZSTD_Dctx_s ZSTD_Dctx; ZSTD_Dctx* ZSTD_createDCtx(void); +size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx); size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx); size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx); diff --git a/programs/Makefile b/programs/Makefile index 7046c087d..981a12a7c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -120,10 +120,22 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-all: test test32 memtest test-zstd: zstd datagen - @echo "*** flush write error test ***" + @echo "\n**** frame concatenation **** " + @echo "hello " > hello.tmp + @echo "world!" > world.tmp + @cat hello.tmp world.tmp > helloworld.tmp + ./zstd hello.tmp > hello.zstd + ./zstd world.tmp > world.zstd + @cat hello.zstd world.zstd > helloworld.zstd + ./zstd -d helloworld.zstd > result.tmp + cat result.tmp + sdiff helloworld.tmp result.tmp + @rm *.tmp *.zstd + @echo frame concatenation test completed + @echo "**** flush write error test **** " echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi - @echo "*** zstd round-trip tests *** " + @echo "**** zstd round-trip tests **** " ./datagen | ./zstd -v | ./zstd -d > $(VOID) ./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID) ./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID) diff --git a/programs/fileio.c b/programs/fileio.c index 5b1ff0625..2da5632bb 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -309,45 +309,18 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* #define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3 -unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename) +unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput, + BYTE* inBuff, size_t inBuffSize, + BYTE* outBuff, size_t outBuffSize, + ZSTD_Dctx* dctx) { - FILE* finput, *foutput; - BYTE* inBuff; - size_t inBuffSize; - BYTE* outBuff, *op, *oend; - size_t outBuffSize; - U32 blockSize = 128 KB; - U32 wNbBlocks = 4; + BYTE* op = outBuff; + BYTE* const oend = outBuff + outBuffSize; U64 filesize = 0; - BYTE* header[MAXHEADERSIZE]; - ZSTD_Dctx* dctx; size_t toRead; size_t sizeCheck; - /* Init */ - FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); - dctx = ZSTD_createDCtx(); - - /* check header */ - toRead = ZSTD_nextSrcSizeToDecompress(dctx); - if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header"); - sizeCheck = fread(header, (size_t)1, toRead, finput); - if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header"); - sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header - if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header"); - - /* Here later : blockSize determination */ - - /* Allocate Memory */ - inBuffSize = blockSize + FIO_blockHeaderSize; - inBuff = (BYTE*)malloc(inBuffSize); - outBuffSize = wNbBlocks * blockSize; - outBuff = (BYTE*)malloc(outBuffSize); - op = outBuff; - oend = outBuff + outBuffSize; - if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory"); - /* Main decompression Loop */ toRead = ZSTD_nextSrcSizeToDecompress(dctx); while (toRead) @@ -380,6 +353,67 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha toRead = ZSTD_nextSrcSizeToDecompress(dctx); } + return filesize; +} + + +unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename) +{ + FILE* finput, *foutput; + BYTE* inBuff=NULL; + size_t inBuffSize = 0; + BYTE* outBuff=NULL; + size_t outBuffSize = 0; + U32 blockSize = 128 KB; + U32 wNbBlocks = 4; + U64 filesize = 0; + BYTE* header[MAXHEADERSIZE]; + ZSTD_Dctx* dctx; + size_t toRead; + size_t sizeCheck; + + + /* Init */ + FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); + dctx = ZSTD_createDCtx(); + + /* for each frame */ + for ( ; ; ) + { + /* check header */ + ZSTD_resetDCtx(dctx); + toRead = ZSTD_nextSrcSizeToDecompress(dctx); + if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header"); + sizeCheck = fread(header, (size_t)1, toRead, finput); + if (sizeCheck==0) break; /* no more input */ + if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header"); + sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header + if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header"); + + /* Here later : blockSize determination */ + + /* Allocate Memory (if needed) */ + { + size_t newInBuffSize = blockSize + FIO_blockHeaderSize; + size_t newOutBuffSize = wNbBlocks * blockSize; + if (newInBuffSize > inBuffSize) + { + free(inBuff); + inBuffSize = newInBuffSize; + inBuff = (BYTE*)malloc(inBuffSize); + } + if (newOutBuffSize > outBuffSize) + { + free(outBuff); + outBuffSize = newOutBuffSize; + outBuff = (BYTE*)malloc(outBuffSize); + } + } + if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory"); + + filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2,"Decoded %llu bytes \n", (long long unsigned)filesize);