lib/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2026-01-06 11:21:19 +03:00
Code Activity

Merge branch 'dev' into newFormats

Browse Source 
Fixed conflicts in zstdmt_compress.c
This commit is contained in:
Yann Collet
2017-09-27 16:39:40 -07:00
parent ea1f50bf73 e45a2aea9b
commit 54a827fff0
24 changed files with 449 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
programs/fileio.c
View File

@@ -37,6 +37,7 @@
# include <io.h>
#endif
#include "bitstream.h"
#include "mem.h"
#include "fileio.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
@@ -347,13 +348,16 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
fileHandle = fopen(fileName, "rb");
if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
fileSize = UTIL_getFileSize(fileName);
if (fileSize > DICTSIZE_MAX)
if (fileSize > DICTSIZE_MAX) {
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */
}
*bufferPtr = malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
if (readSize!=fileSize)
EXM_THROW(35, "Error reading dictionary file %s", fileName);
}
fclose(fileHandle);
return (size_t)fileSize;
}
@@ -970,7 +974,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
char* dstFileName = (char*)malloc(FNSPACE);
size_t const suffixSize = suffix ? strlen(suffix) : 0;
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
int const isRegularFile = (nbFiles != 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]);
int const isRegularFile = (nbFiles > 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]); /* won't write frame content size when nbFiles > 1 */
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
/* init */
@@ -1167,6 +1171,35 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
return 0;
}
static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName)
{
ZSTD_frameHeader header;
/* No special help for these errors */
if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge)
return;
/* Try to decode the frame header */
ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
if (ret == 0) {
U32 const windowSize = (U32)header.windowSize;
U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0);
U32 const windowMB = (windowSize >> 20) + (windowSize & ((1 MB) - 1));
assert(header.windowSize <= (U64)((U32)-1));
assert(g_memLimit > 0);
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
srcFileName, header.windowSize, g_memLimit);
if (windowLog <= ZSTD_WINDOWLOG_MAX) {
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
srcFileName, windowLog, windowMB);
return;
}
} else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) {
DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n",
srcFileName, ZSTD_getErrorName(ret));
return;
}
DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n",
srcFileName, ZSTD_WINDOWLOG_MAX);
}
/** FIO_decompressFrame() :
* @return : size of decoded zstd frame, or an error code
@@ -1183,8 +1216,8 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
ZSTD_resetDStream(ress->dctx);
if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */
/* Header loading (optional, saves one loop) */
{ size_t const toRead = 9;
/* Header loading : ensures ZSTD_getFrameHeader() will succeed */
{ size_t const toRead = ZSTD_FRAMEHEADERSIZE_MAX;
if (ress->srcBufferLoaded < toRead)
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
}
@@ -1197,6 +1230,7 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(readSizeHint));
FIO_zstdErrorHelp(ress, readSizeHint, srcFileName);
return FIO_ERROR_FRAME_DECODING;
}
@@ -1700,10 +1734,11 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
typedef struct {
int numActualFrames;
int numSkippableFrames;
unsigned long long decompressedSize;
U64 decompressedSize;
int decompUnavailable;
unsigned long long compressedSize;
U64 compressedSize;
int usesCheck;
U32 nbFiles;
} fileInfo_t;
/** getFileInfo() :
@@ -1720,14 +1755,16 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
DISPLAY("Error: could not open source file %s\n", inFileName);
return 3;
}
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
info->compressedSize = UTIL_getFileSize(inFileName);
/* begin analyzing frame */
for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_frameHeaderSize_min) {
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
if ( feof(srcFile)
&& (numBytesRead == 0)
&& (info->compressedSize > 0) ) {
break;
}
else if (feof(srcFile)) {
@@ -1829,6 +1866,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
}
} /* end analyzing frame */
fclose(srcFile);
info->nbFiles = 1;
return detectError;
}
@@ -1841,25 +1879,28 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
if (displayLevel <= 2) {
if (!info->decompUnavailable) {
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
info->numSkippableFrames, info->numActualFrames,
DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
info->numSkippableFrames + info->numActualFrames,
info->numSkippableFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, inFileName);
} else {
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
info->numSkippableFrames, info->numActualFrames,
compressedSizeUnit, checkString, inFileName);
DISPLAYOUT("%6d %5d %7.2f %2s %5s %s\n",
info->numSkippableFrames + info->numActualFrames,
info->numSkippableFrames,
compressedSizeUnit, unitStr,
checkString, inFileName);
}
} else {
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
compressedSizeUnit, unitStr, info->compressedSize);
compressedSizeUnit, unitStr,
(unsigned long long)info->compressedSize);
if (!info->decompUnavailable) {
DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
decompressedSizeUnit, unitStr, info->decompressedSize);
decompressedSizeUnit, unitStr,
(unsigned long long)info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", ratio);
}
DISPLAYOUT("Check: %s\n", checkString);
@@ -1867,33 +1908,40 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
}
}
static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
{
fileInfo_t total;
total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames;
total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames;
total.compressedSize = fi1.compressedSize + fi2.compressedSize;
total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize;
total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable;
total.usesCheck = fi1.usesCheck & fi2.usesCheck;
total.nbFiles = fi1.nbFiles + fi2.nbFiles;
return total;
}
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
/* initialize info to avoid warnings */
fileInfo_t info;
memset(&info, 0, sizeof(info));
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
{
int const error = getFileInfo(&info, inFileName);
{ int const error = getFileInfo(&info, inFileName);
if (error == 1) {
/* display error, but provide output */
DISPLAY("An error occurred with getting file info\n");
DISPLAY("An error occurred while getting file info \n");
}
else if (error == 2) {
DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
DISPLAYOUT("File %s not compressed by zstd \n", inFileName);
if (displayLevel > 2) DISPLAYOUT("\n");
return 1;
}
else if (error == 3) {
/* error occurred with opening the file */
if (displayLevel > 2) {
DISPLAYOUT("\n");
}
/* error occurred while opening the file */
if (displayLevel > 2) DISPLAYOUT("\n");
return 1;
}
displayInfo(inFileName, &info, displayLevel);
*total = FIO_addFInfo(*total, info);
return error;
}
}
@@ -1903,15 +1951,36 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
DISPLAYOUT("No files given\n");
return 0;
}
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Printing information about compressed files\n");
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Number of files listed: %u\n", numFiles);
{
int error = 0;
DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n");
{ int error = 0;
unsigned u;
fileInfo_t total;
memset(&total, 0, sizeof(total));
total.usesCheck = 1;
for (u=0; u<numFiles;u++) {
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
error |= FIO_listFile(&total, filenameTable[u], displayLevel);
}
if (numFiles > 1) {
unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
double const compressedSizeUnit = (double)total.compressedSize / unit;
double const decompressedSizeUnit = (double)total.decompressedSize / unit;
double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize;
const char* const checkString = (total.usesCheck ? "XXH64" : "");
DISPLAYOUT("----------------------------------------------------------------- \n");
if (total.decompUnavailable) {
DISPLAYOUT("%6d %5d %7.2f %2s %5s %u files\n",
total.numSkippableFrames + total.numActualFrames,
total.numSkippableFrames,
compressedSizeUnit, unitStr,
checkString, total.nbFiles);
} else {
DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n",
total.numSkippableFrames + total.numActualFrames,
total.numSkippableFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, total.nbFiles);
}
}
return error;
}

18
programs/zstd.1
View File

@@ -104,8 +104,11 @@ Display information related to a zstd compressed file, such as size, ratio, and
unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
.
.TP
\fB\-\-long\fR
enables long distance matching\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance (up to the maximum window size, 128 MiB)\.
\fB\-\-long[=#]\fR
enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
.
.IP
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.
.TP
\fB\-T#\fR, \fB\-\-threads=#\fR
@@ -190,6 +193,10 @@ Dictionary saved into \fBfile\fR (default name: dictionary)\.
Limit dictionary to specified size (default: 112640)\.
.
.TP
\fB\-B#\fR
Split input files in blocks of size # (default: no split)
.
.TP
\fB\-\-dictID=#\fR
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
.
@@ -267,7 +274,10 @@ There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fas
Specify the maximum number of bits for a match distance\.
.
.IP
The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 27 (128 MiB)\.
The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32\-bit platforms and 31 (2 GiB) on 64\-bit platforms\.
.
.IP
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.
.TP
\fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR
@@ -340,7 +350,7 @@ Bigger hash tables usually improve compression ratio at the expense of more memo
The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
.
.TP
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmSlen\fR=\fIldmslen\fR
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
Specify the minimum searched length of a match for long distance matching\.
.
.IP

20
programs/zstd.1.md
View File

@@ -105,12 +105,16 @@ the last one takes effect.
* `--ultra`:
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
Note that decompression will also require more memory when using these levels.
* `--long`:
enables long distance matching.
* `--long[=#]`:
enables long distance matching with `#` `windowLog`, if not `#` is not
present it defaults to `27`.
This increases the window size (`windowLog`) and memory usage for both the
compressor and decompressor. This setting is designed to improve the
compression ratio for files with long matches at a large distance
(up to the maximum window size, 128 MiB).
compressor and decompressor.
This setting is designed to improve the compression ratio for files with
long matches at a large distance.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor.
* `-T#`, `--threads=#`:
Compress using `#` threads (default: 1).
If `#` is 0, attempt to detect and use the number of physical CPU cores.
@@ -275,7 +279,11 @@ The list of available _options_:
The higher number of increases the chance to find a match which usually
improves compression ratio.
It also increases memory requirements for the compressor and decompressor.
The minimum _wlog_ is 10 (1 KiB) and the maximum is 27 (128 MiB).
The minimum _wlog_ is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32-bit
platforms and 31 (2 GiB) on 64-bit platforms.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor.
- `hashLog`=_hlog_, `hlog`=_hlog_:
Specify the maximum number of bits for a hash table.

27
programs/zstdcli.c
View File

@@ -72,6 +72,7 @@ static const char* g_defaultDictName = "dictionary";
static const unsigned g_defaultMaxDictSize = 110 KB;
static const int g_defaultDictCLevel = 3;
static const unsigned g_defaultSelectivityLevel = 9;
static const unsigned g_defaultMaxWindowLog = 27;
#define OVERLAP_LOG_DEFAULT 9999
#define LDM_PARAM_DEFAULT 9999 /* Default for parameters where 0 is valid */
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
@@ -129,7 +130,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -l : print information about zstd compressed files \n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAY( "--long : enable long distance matching (requires more memory)\n");
DISPLAY( "--long[=#] : enable long distance matching with given window log (default : %u)\n", g_defaultMaxWindowLog);
#ifdef ZSTD_MULTITHREAD
DISPLAY( " -T# : use # threads for compression (default:1) \n");
DISPLAY( " -B# : select size of each job (default:0==automatic) \n");
@@ -459,7 +460,6 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
if (!strcmp(argument, "--long")) { ldmFlag = 1; continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
@@ -514,6 +514,22 @@ int main(int argCount, const char* argv[])
if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
if (longCommandWArg(&argument, "--long")) {
unsigned ldmWindowLog = 0;
ldmFlag = 1;
/* Parse optional window log */
if (*argument == '=') {
++argument;
ldmWindowLog = readU32FromChar(&argument);
} else if (*argument != 0) {
/* Invalid character following --long */
CLEAN_RETURN(badusage(programName));
}
/* Only set windowLog if not already set by --zstd */
if (compressionParams.windowLog == 0)
compressionParams.windowLog = ldmWindowLog;
continue;
}
/* fall-through, will trigger bad_usage() later on */
}
@@ -830,6 +846,13 @@ int main(int argCount, const char* argv[])
#endif
} else { /* decompression or test */
#ifndef ZSTD_NODECOMPRESS
if (memLimit == 0) {
if (compressionParams.windowLog == 0)
memLimit = (U32)1 << g_defaultMaxWindowLog;
else {
memLimit = (U32)1 << (compressionParams.windowLog & 31);
}
}
FIO_setMemLimit(memLimit);
if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);