diff --git a/programs/fileio.c b/programs/fileio.c index b3f324ac6..cdfe6a3dc 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -130,6 +130,8 @@ static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } static U32 g_maxWLog = 23; void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; } +static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ +void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; } /*-************************************* @@ -178,6 +180,10 @@ static FILE* FIO_openDstFile(const char* dstFileName) DISPLAYLEVEL(4,"Using stdout for output\n"); f = stdout; SET_BINARY_MODE(stdout); + if (g_sparseFileSupport==1) { + g_sparseFileSupport = 0; + DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); + } } else { if (!g_overwrite) { /* Check if destination file already exists */ f = fopen( dstFileName, "rb" ); @@ -189,8 +195,7 @@ static FILE* FIO_openDstFile(const char* dstFileName) return 0; } DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName); - { - int ch = getchar(); + { int ch = getchar(); if ((ch!='Y') && (ch!='y')) { DISPLAY(" not overwritten \n"); return 0; @@ -512,6 +517,81 @@ static void FIO_freeDResources(dRess_t ress) } +/** FIO_fwriteSparse() : +* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ +static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +{ + const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ + size_t bufferSizeT = bufferSize / sizeof(size_t); + const size_t* const bufferTEnd = bufferT + bufferSizeT; + const size_t* ptrT = bufferT; + static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ + + if (!g_sparseFileSupport) { /* normal write */ + size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); + if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block"); + return 0; + } + + /* avoid int overflow */ + if (storedSkips > 1 GB) { + int const seekResult = fseek(file, 1 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); + storedSkips -= 1 GB; + } + + while (ptrT < bufferTEnd) { + size_t seg0SizeT = segmentSizeT; + size_t nb0T; + + /* count leading zeros */ + if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; + bufferSizeT -= seg0SizeT; + for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeof(size_t)); + + if (nb0T != seg0SizeT) { /* not all 0s */ + int const seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + seg0SizeT -= nb0T; + ptrT += nb0T; + { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file); + if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block"); + } } + ptrT += seg0SizeT; + } + + { static size_t const maskT = sizeof(size_t)-1; + if (bufferSize & maskT) { /* size not multiple of sizeof(size_t) : implies end of block */ + const char* const restStart = (const char*)bufferTEnd; + const char* restPtr = restStart; + size_t restSize = bufferSize & maskT; + const char* const restEnd = restStart + restSize; + for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) { + int seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); + if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block"); + } } } } + + return storedSkips; +} + +static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) +{ + if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */ + int const seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); + { const char lastZeroByte[1] = { 0 }; + size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); + } } +} + /** FIO_decompressFrame() : @return : size of decoded frame */ @@ -520,6 +600,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, { U64 frameSize = 0; size_t readSize; + U32 storedSkips = 0; ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); @@ -538,8 +619,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress, readSize -= inSize; /* Write block */ - { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); - if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); } + storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, decodedSize, storedSkips); frameSize += decodedSize; DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); @@ -553,6 +633,8 @@ unsigned long long FIO_decompressFrame(dRess_t ress, EXM_THROW(35, "Read error"); } + FIO_fwriteSparseEnd(foutput, storedSkips); + return frameSize; } diff --git a/programs/fileio.h b/programs/fileio.h index d5aae449b..6e7912380 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -46,7 +46,8 @@ extern "C" { ***************************************/ void FIO_overwriteMode(void); void FIO_setNotificationLevel(unsigned level); -void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ +void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */ +void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ /*-************************************* diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 3bb4db70a..3fdc558bd 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -132,6 +132,7 @@ static int usage_advanced(const char* programName) #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); #endif + DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); #ifndef ZSTD_NODICT DISPLAY( "\n"); DISPLAY( "Dictionary builder :\n"); @@ -229,6 +230,8 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */ if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } + if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } + if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } /* '-' means stdin/stdout */ if (!strcmp(argument, "-")){