From 254c5b1692c6b9457fae4a45a95e1eef13cf028d Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 21 Sep 2016 14:29:47 -0700 Subject: [PATCH] [pzstd] Make CLI compatible with zstd --- contrib/pzstd/Options.cpp | 485 +++++++++++++++++++------- contrib/pzstd/Options.h | 48 +-- contrib/pzstd/Pzstd.cpp | 175 ++++++++-- contrib/pzstd/Pzstd.h | 20 +- contrib/pzstd/main.cpp | 16 +- contrib/pzstd/test/OptionsTest.cpp | 526 ++++++++++++++++++++++++----- contrib/pzstd/test/PzstdTest.cpp | 13 +- contrib/pzstd/test/RoundTrip.h | 17 +- contrib/pzstd/utils/FileSystem.h | 16 + 9 files changed, 1017 insertions(+), 299 deletions(-) diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp index 122f4fb36..055a07907 100644 --- a/contrib/pzstd/Options.cpp +++ b/contrib/pzstd/Options.cpp @@ -7,182 +7,419 @@ * of patent rights can be found in the PATENTS file in the same directory. */ #include "Options.h" +#include "utils/ScopeGuard.h" +#include +#include #include #include #include +#include +#include + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || \ + defined(__CYGWIN__) +#include /* _isatty */ +#define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || \ + defined(_POSIX_SOURCE) || \ + (defined(__APPLE__) && \ + defined( \ + __MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ \ + */ +#include /* isatty */ +#define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#else +#define IS_CONSOLE(stdStream) 0 +#endif +#endif namespace pzstd { namespace { -unsigned parseUnsigned(const char* arg) { +unsigned defaultNumThreads() { +#ifdef PZSTD_NUM_THREADS + return PZSTD_NUM_THREADS; +#else + return std::thread::hardware_concurrency(); +#endif +} + +unsigned parseUnsigned(const char **arg) { unsigned result = 0; - while (*arg >= '0' && *arg <= '9') { + while (**arg >= '0' && **arg <= '9') { result *= 10; - result += *arg - '0'; - ++arg; + result += **arg - '0'; + ++(*arg); } return result; } -const std::string zstdExtension = ".zst"; -constexpr unsigned defaultCompressionLevel = 3; -constexpr unsigned maxNonUltraCompressionLevel = 19; +const char *getArgument(const char *options, const char **argv, int &i, + int argc) { + if (options[1] != 0) { + return options + 1; + } + ++i; + if (i == argc) { + std::fprintf(stderr, "Option -%c requires an argument, but none provided\n", + *options); + return nullptr; + } + return argv[i]; +} + +const std::string kZstdExtension = ".zst"; +constexpr char kStdIn[] = "-"; +constexpr char kStdOut[] = "-"; +constexpr unsigned kDefaultCompressionLevel = 3; +constexpr unsigned kMaxNonUltraCompressionLevel = 19; + +#ifdef _WIN32 +const char nullOutput[] = "nul"; +#else +const char nullOutput[] = "/dev/null"; +#endif + +void notSupported(const char *option) { + std::fprintf(stderr, "Operation not supported: %s\n", option); +} void usage() { std::fprintf(stderr, "Usage:\n"); - std::fprintf(stderr, "\tpzstd [args] FILE\n"); + std::fprintf(stderr, " pzstd [args] [FILE(s)]\n"); std::fprintf(stderr, "Parallel ZSTD options:\n"); - std::fprintf(stderr, "\t-n/--num-threads #: Number of threads to spawn\n"); - std::fprintf(stderr, "\t-p/--pzstd-headers: Write pzstd headers to enable parallel decompression\n"); + std::fprintf(stderr, " -p, --processes # : number of threads to use for (de)compression (default:%d)\n", defaultNumThreads()); std::fprintf(stderr, "ZSTD options:\n"); - std::fprintf(stderr, "\t-u/--ultra : enable levels beyond %i, up to %i (requires more memory)\n", maxNonUltraCompressionLevel, ZSTD_maxCLevel()); - std::fprintf(stderr, "\t-h/--help : display help and exit\n"); - std::fprintf(stderr, "\t-V/--version : display version number and exit\n"); - std::fprintf(stderr, "\t-d/--decompress : decompression\n"); - std::fprintf(stderr, "\t-f/--force : overwrite output\n"); - std::fprintf(stderr, "\t-o/--output file : result stored into `file`\n"); - std::fprintf(stderr, "\t-c/--stdout : write output to standard output\n"); - std::fprintf(stderr, "\t-# : # compression level (1-%d, default:%d)\n", maxNonUltraCompressionLevel, defaultCompressionLevel); + std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel); + std::fprintf(stderr, " -d, --decompress : decompression\n"); + std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n"); + std::fprintf(stderr, " -f, --force : overwrite output without prompting\n"); + std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n"); + std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n"); + std::fprintf(stderr, " -h, --help : display help and exit\n"); + std::fprintf(stderr, " -V, --version : display version number and exit\n"); + std::fprintf(stderr, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n"); + std::fprintf(stderr, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n"); + std::fprintf(stderr, " -c, --stdout : force wrtie to standard output, even if it is the console\n"); +#ifdef UTIL_HAS_CREATEFILELIST + std::fprintf(stderr, " -r : operate recursively on directories\n"); +#endif + std::fprintf(stderr, " --ultra : enable levels beyond %i, up to %i (requires more memory)\n", kMaxNonUltraCompressionLevel, ZSTD_maxCLevel()); + std::fprintf(stderr, " -C, --check : integrity check (default)\n"); + std::fprintf(stderr, " --no-check : no integrity check\n"); + std::fprintf(stderr, " -t, --test : test compressed file integrity\n"); + std::fprintf(stderr, " -- : all arguments after \"--\" are treated as files\n"); } } // anonymous namespace Options::Options() - : numThreads(0), - maxWindowLog(23), - compressionLevel(defaultCompressionLevel), - decompress(false), - overwrite(false), - pzstdHeaders(false) {} + : numThreads(defaultNumThreads()), maxWindowLog(23), + compressionLevel(kDefaultCompressionLevel), decompress(false), + overwrite(false), keepSource(true), writeMode(WriteMode::Auto), + checksum(true), verbosity(2) {} -bool Options::parse(int argc, const char** argv) { +Options::Status Options::parse(int argc, const char **argv) { + bool test = false; + bool recursive = false; bool ultra = false; + bool forceStdout = false; + // Local copy of input files, which are pointers into argv. + std::vector localInputFiles; for (int i = 1; i < argc; ++i) { - const char* arg = argv[i]; - // Arguments with a short option - char option = 0; - if (!std::strcmp(arg, "--num-threads")) { - option = 'n'; - } else if (!std::strcmp(arg, "--pzstd-headers")) { - option = 'p'; - } else if (!std::strcmp(arg, "--ultra")) { - option = 'u'; - } else if (!std::strcmp(arg, "--version")) { - option = 'V'; - } else if (!std::strcmp(arg, "--help")) { - option = 'h'; - } else if (!std::strcmp(arg, "--decompress")) { - option = 'd'; - } else if (!std::strcmp(arg, "--force")) { - option = 'f'; - } else if (!std::strcmp(arg, "--output")) { - option = 'o'; - } else if (!std::strcmp(arg, "--stdout")) { - option = 'c'; - }else if (arg[0] == '-' && arg[1] != 0) { - // Parse the compression level or short option - if (arg[1] >= '0' && arg[1] <= '9') { - compressionLevel = parseUnsigned(arg + 1); - continue; - } - option = arg[1]; - } else if (inputFile.empty()) { - inputFile = arg; + const char *arg = argv[i]; + // Protect against empty arguments + if (arg[0] == 0) { continue; - } else { - std::fprintf(stderr, "Invalid argument: %s.\n", arg); - return false; } - - switch (option) { - case 'n': - if (++i == argc) { - std::fprintf(stderr, "Invalid argument: -n requires an argument.\n"); - return false; - } - numThreads = parseUnsigned(argv[i]); - if (numThreads == 0) { - std::fprintf(stderr, "Invalid argument: # of threads must be > 0.\n"); - return false; - } - break; - case 'p': - pzstdHeaders = true; - break; - case 'u': + // Everything after "--" is an input file + if (!std::strcmp(arg, "--")) { + ++i; + std::copy(argv + i, argv + argc, std::back_inserter(localInputFiles)); + break; + } + // Long arguments that don't have a short option + { + bool isLongOption = true; + if (!std::strcmp(arg, "--rm")) { + keepSource = false; + } else if (!std::strcmp(arg, "--ultra")) { ultra = true; maxWindowLog = 0; - break; - case 'V': - std::fprintf(stderr, "ZSTD version: %s.\n", ZSTD_VERSION_STRING); - return false; + } else if (!std::strcmp(arg, "--no-check")) { + checksum = false; + } else if (!std::strcmp(arg, "--sparse")) { + writeMode = WriteMode::Sparse; + notSupported("Sparse mode"); + return Status::Failure; + } else if (!std::strcmp(arg, "--no-sparse")) { + writeMode = WriteMode::Regular; + notSupported("Sparse mode"); + return Status::Failure; + } else if (!std::strcmp(arg, "--dictID")) { + notSupported(arg); + return Status::Failure; + } else if (!std::strcmp(arg, "--no-dictID")) { + notSupported(arg); + return Status::Failure; + } else { + isLongOption = false; + } + if (isLongOption) { + continue; + } + } + // Arguments with a short option simply set their short option. + const char *options = nullptr; + if (!std::strcmp(arg, "--processes")) { + options = "p"; + } else if (!std::strcmp(arg, "--version")) { + options = "V"; + } else if (!std::strcmp(arg, "--help")) { + options = "h"; + } else if (!std::strcmp(arg, "--decompress")) { + options = "d"; + } else if (!std::strcmp(arg, "--force")) { + options = "f"; + } else if (!std::strcmp(arg, "--stdout")) { + options = "c"; + } else if (!std::strcmp(arg, "--keep")) { + options = "k"; + } else if (!std::strcmp(arg, "--verbose")) { + options = "v"; + } else if (!std::strcmp(arg, "--quiet")) { + options = "q"; + } else if (!std::strcmp(arg, "--check")) { + options = "C"; + } else if (!std::strcmp(arg, "--test")) { + options = "t"; + } else if (arg[0] == '-' && arg[1] != 0) { + options = arg + 1; + } else { + localInputFiles.emplace_back(arg); + continue; + } + assert(options != nullptr); + + bool finished = false; + while (!finished && *options != 0) { + // Parse the compression level + if (*options >= '0' && *options <= '9') { + compressionLevel = parseUnsigned(&options); + continue; + } + + switch (*options) { case 'h': + case 'H': usage(); - return false; + return Status::Message; + case 'V': + std::fprintf(stderr, "PZSTD version: %s.\n", ZSTD_VERSION_STRING); + return Status::Message; + case 'p': { + finished = true; + const char *optionArgument = getArgument(options, argv, i, argc); + if (optionArgument == nullptr) { + return Status::Failure; + } + if (*optionArgument < '0' || *optionArgument > '9') { + std::fprintf(stderr, "Option -p expects a number, but %s provided\n", + optionArgument); + return Status::Failure; + } + numThreads = parseUnsigned(&optionArgument); + if (*optionArgument != 0) { + std::fprintf(stderr, + "Option -p expects a number, but %u%s provided\n", + numThreads, optionArgument); + return Status::Failure; + } + break; + } + case 'o': { + finished = true; + const char *optionArgument = getArgument(options, argv, i, argc); + if (optionArgument == nullptr) { + return Status::Failure; + } + outputFile = optionArgument; + break; + } + case 'C': + checksum = true; + break; + case 'k': + keepSource = true; + break; case 'd': decompress = true; break; case 'f': overwrite = true; + forceStdout = true; break; - case 'o': - if (++i == argc) { - std::fprintf(stderr, "Invalid argument: -o requires an argument.\n"); - return false; - } - outputFile = argv[i]; + case 't': + test = true; + decompress = true; break; +#ifdef UTIL_HAS_CREATEFILELIST + case 'r': + recursive = true; + break; +#endif case 'c': - outputFile = '-'; + outputFile = kStdOut; + forceStdout = true; break; + case 'v': + ++verbosity; + break; + case 'q': + --verbosity; + // Ignore them for now + break; + // Unsupported options from Zstd + case 'D': + case 's': + notSupported("Zstd dictionaries."); + return Status::Failure; + case 'b': + case 'e': + case 'i': + case 'B': + notSupported("Zstd benchmarking options."); + return Status::Failure; default: - std::fprintf(stderr, "Invalid argument: %s.\n", arg); - return false; - } - } - // Determine input file if not specified - if (inputFile.empty()) { - inputFile = "-"; - } - // Determine output file if not specified - if (outputFile.empty()) { - if (inputFile == "-") { - outputFile = "-"; - } else { - // Attempt to add/remove zstd extension from the input file - if (decompress) { - int stemSize = inputFile.size() - zstdExtension.size(); - if (stemSize > 0 && inputFile.substr(stemSize) == zstdExtension) { - outputFile = inputFile.substr(0, stemSize); - } else { - std::fprintf( - stderr, "Invalid argument: Unable to determine output file.\n"); - return false; - } - } else { - outputFile = inputFile + zstdExtension; + std::fprintf(stderr, "Invalid argument: -%c\n", *options); + return Status::Failure; } + if (!finished) { + ++options; + } + } // while (*options != 0); + } // for (int i = 1; i < argc; ++i); + + // Input file defaults to standard input if not provided. + if (localInputFiles.empty()) { + localInputFiles.emplace_back(kStdIn); + } + + // Check validity of input files + if (localInputFiles.size() > 1) { + const auto it = std::find(localInputFiles.begin(), localInputFiles.end(), + std::string{kStdIn}); + if (it != localInputFiles.end()) { + std::fprintf( + stderr, + "Cannot specify standard input when handling multiple files\n"); + return Status::Failure; } } + if (localInputFiles.size() > 1 || recursive) { + if (!outputFile.empty() && outputFile != nullOutput) { + std::fprintf( + stderr, + "Cannot specify an output file when handling multiple inputs\n"); + return Status::Failure; + } + } + + // Translate input files/directories into files to (de)compress + if (recursive) { + char *scratchBuffer = nullptr; + unsigned numFiles = 0; + const char **files = + UTIL_createFileList(localInputFiles.data(), localInputFiles.size(), + &scratchBuffer, &numFiles); + if (files == nullptr) { + std::fprintf(stderr, "Error traversing directories\n"); + return Status::Failure; + } + auto guard = + makeScopeGuard([&] { UTIL_freeFileList(files, scratchBuffer); }); + if (numFiles == 0) { + std::fprintf(stderr, "No files found\n"); + return Status::Failure; + } + inputFiles.resize(numFiles); + std::copy(files, files + numFiles, inputFiles.begin()); + } else { + inputFiles.resize(localInputFiles.size()); + std::copy(localInputFiles.begin(), localInputFiles.end(), + inputFiles.begin()); + } + localInputFiles.clear(); + assert(!inputFiles.empty()); + + // If reading from standard input, default to standard output + if (inputFiles[0] == kStdIn && outputFile.empty()) { + assert(inputFiles.size() == 1); + outputFile = "-"; + } + + if (inputFiles[0] == kStdIn && IS_CONSOLE(stdin)) { + assert(inputFiles.size() == 1); + std::fprintf(stderr, "Cannot read input from interactive console\n"); + return Status::Failure; + } + if (outputFile == "-" && IS_CONSOLE(stdout) && !(forceStdout && decompress)) { + std::fprintf(stderr, "Will not write to console stdout unless -c or -f is " + "specified and decompressing\n"); + return Status::Failure; + } + // Check compression level { - unsigned maxCLevel = ultra ? ZSTD_maxCLevel() : maxNonUltraCompressionLevel; - if (compressionLevel > maxCLevel) { - std::fprintf( - stderr, "Invalid compression level %u.\n", compressionLevel); - return false; + unsigned maxCLevel = + ultra ? ZSTD_maxCLevel() : kMaxNonUltraCompressionLevel; + if (compressionLevel > maxCLevel || compressionLevel == 0) { + std::fprintf(stderr, "Invalid compression level %u.\n", compressionLevel); + return Status::Failure; } } + // Check that numThreads is set if (numThreads == 0) { - numThreads = std::thread::hardware_concurrency(); - if (numThreads == 0) { - std::fprintf(stderr, "Invalid arguments: # of threads not specified " - "and unable to determine hardware concurrency.\n"); - return false; - } + std::fprintf(stderr, "Invalid arguments: # of threads not specified " + "and unable to determine hardware concurrency.\n"); + return Status::Failure; + } + + // Modify verbosity + // If we are piping input and output, turn off interaction + if (inputFiles[0] == kStdIn && outputFile == kStdOut && verbosity == 2) { + verbosity = 1; + } + // If we are in multi-file mode, turn off interaction + if (inputFiles.size() > 1 && verbosity == 2) { + verbosity = 1; + } + + // Set options for test mode + if (test) { + outputFile = nullOutput; + keepSource = true; + } + return Status::Success; +} + +std::string Options::getOutputFile(const std::string &inputFile) const { + if (!outputFile.empty()) { + return outputFile; + } + // Attempt to add/remove zstd extension from the input file + if (decompress) { + int stemSize = inputFile.size() - kZstdExtension.size(); + if (stemSize > 0 && inputFile.substr(stemSize) == kZstdExtension) { + return inputFile.substr(0, stemSize); + } else { + return ""; + } + } else { + return inputFile + kZstdExtension; } - return true; } } diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h index 47c5f78a6..97c3885ec 100644 --- a/contrib/pzstd/Options.h +++ b/contrib/pzstd/Options.h @@ -14,47 +14,55 @@ #include #include +#include namespace pzstd { struct Options { + enum class WriteMode { Regular, Auto, Sparse }; + unsigned numThreads; unsigned maxWindowLog; unsigned compressionLevel; bool decompress; - std::string inputFile; + std::vector inputFiles; std::string outputFile; bool overwrite; - bool pzstdHeaders; + bool keepSource; + WriteMode writeMode; + bool checksum; + int verbosity; + + enum class Status { + Success, // Successfully parsed options + Failure, // Failure to parse options + Message // Options specified to print a message (e.g. "-h") + }; Options(); - Options( - unsigned numThreads, - unsigned maxWindowLog, - unsigned compressionLevel, - bool decompress, - const std::string& inputFile, - const std::string& outputFile, - bool overwrite, - bool pzstdHeaders) - : numThreads(numThreads), - maxWindowLog(maxWindowLog), - compressionLevel(compressionLevel), - decompress(decompress), - inputFile(inputFile), - outputFile(outputFile), - overwrite(overwrite), - pzstdHeaders(pzstdHeaders) {} + Options(unsigned numThreads, unsigned maxWindowLog, unsigned compressionLevel, + bool decompress, std::vector inputFiles, + std::string outputFile, bool overwrite, bool keepSource, + WriteMode writeMode, bool checksum, int verbosity) + : numThreads(numThreads), maxWindowLog(maxWindowLog), + compressionLevel(compressionLevel), decompress(decompress), + inputFiles(std::move(inputFiles)), outputFile(std::move(outputFile)), + overwrite(overwrite), keepSource(keepSource), writeMode(writeMode), + checksum(checksum), verbosity(verbosity) {} - bool parse(int argc, const char** argv); + Status parse(int argc, const char **argv); ZSTD_parameters determineParameters() const { ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0); + params.fParams.contentSizeFlag = 1; + params.fParams.checksumFlag = checksum; if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) { params.cParams.windowLog = maxWindowLog; params.cParams = ZSTD_adjustCParams(params.cParams, 0, 0); } return params; } + + std::string getOutputFile(const std::string &inputFile) const; }; } diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 87c4c202f..fceb49a7c 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -19,6 +19,15 @@ #include #include +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _O_BINARY */ +# include /* _setmode, _isatty */ +# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); } +#else +# include /* isatty */ +# define SET_BINARY_MODE(file) +#endif + namespace pzstd { namespace { @@ -31,40 +40,25 @@ const std::string nullOutput = "/dev/null"; using std::size_t; -size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) { - // Open the input file and attempt to determine its size - FILE* inputFd = stdin; - std::uintmax_t inputSize = 0; - if (options.inputFile != "-") { - inputFd = std::fopen(options.inputFile.c_str(), "rb"); - if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) { - return 0; - } - std::error_code ec; - inputSize = file_size(options.inputFile, ec); - if (ec) { - inputSize = 0; - } +static std::uintmax_t fileSizeOrZero(const std::string &file) { + if (file == "-") { + return 0; } - auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); }); - - // Check if the output file exists and then open it - FILE* outputFd = stdout; - if (options.outputFile != "-") { - if (!options.overwrite && options.outputFile != nullOutput) { - outputFd = std::fopen(options.outputFile.c_str(), "rb"); - if (!errorHolder.check(outputFd == nullptr, "Output file exists")) { - return 0; - } - } - outputFd = std::fopen(options.outputFile.c_str(), "wb"); - if (!errorHolder.check( - outputFd != nullptr, "Failed to open output file")) { - return 0; - } + std::error_code ec; + auto size = file_size(file, ec); + if (ec) { + size = 0; } - auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); }); + return size; +} +static size_t handleOneInput(const Options &options, + const std::string &inputFile, + FILE* inputFd, + const std::string &outputFile, + FILE* outputFd, + ErrorHolder &errorHolder) { + auto inputSize = fileSizeOrZero(inputFile); // WorkQueue outlives ThreadPool so in the case of error we are certain // we don't accidently try to call push() on it after it is destroyed. WorkQueue> outs{2 * options.numThreads}; @@ -89,21 +83,128 @@ size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) { options.determineParameters()); }); // Start writing - bytesWritten = - writeFile(errorHolder, outs, outputFd, options.pzstdHeaders); + bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress); } else { // Add a job that reads the input and starts all the decompression jobs executor.add([&errorHolder, &outs, &executor, inputFd] { asyncDecompressFrames(errorHolder, outs, executor, inputFd); }); // Start writing - bytesWritten = writeFile( - errorHolder, outs, outputFd, /* writeSkippableFrames */ false); + bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress); } } return bytesWritten; } +static FILE *openInputFile(const std::string &inputFile, + ErrorHolder &errorHolder) { + if (inputFile == "-") { + SET_BINARY_MODE(stdin); + return stdin; + } + auto inputFd = std::fopen(inputFile.c_str(), "rb"); + if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) { + return nullptr; + } + return inputFd; +} + +static FILE *openOutputFile(const Options &options, + const std::string &outputFile, + ErrorHolder &errorHolder) { + if (outputFile == "-") { + SET_BINARY_MODE(stdout); + return stdout; + } + // Check if the output file exists and then open it + if (!options.overwrite && outputFile != nullOutput) { + auto outputFd = std::fopen(outputFile.c_str(), "rb"); + if (outputFd != nullptr) { + std::fclose(outputFd); + if (options.verbosity <= 1) { + errorHolder.setError("Output file exists"); + return nullptr; + } + std::fprintf( + stderr, + "pzstd: %s already exists; do you wish to overwrite (y/n) ? ", + outputFile.c_str()); + int c = getchar(); + if (c != 'y' && c != 'Y') { + errorHolder.setError("Not overwritten"); + return nullptr; + } + } + } + auto outputFd = std::fopen(outputFile.c_str(), "wb"); + if (!errorHolder.check( + outputFd != nullptr, "Failed to open output file")) { + return 0; + } + return outputFd; +} + +int pzstdMain(const Options &options) { + int returnCode = 0; + for (const auto& input : options.inputFiles) { + // Setup the error holder + ErrorHolder errorHolder; + auto printErrorGuard = makeScopeGuard([&] { + if (errorHolder.hasError()) { + returnCode = 1; + if (options.verbosity > 0) { + std::fprintf(stderr, "pzstd: %s: %s.\n", input.c_str(), + errorHolder.getError().c_str()); + } + } else { + + } + }); + // Open the input file + auto inputFd = openInputFile(input, errorHolder); + if (inputFd == nullptr) { + continue; + } + auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); }); + // Open the output file + auto outputFile = options.getOutputFile(input); + if (!errorHolder.check(outputFile != "", + "Input file does not have extension .zst")) { + continue; + } + auto outputFd = openOutputFile(options, outputFile, errorHolder); + if (outputFd == nullptr) { + continue; + } + auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); }); + // (de)compress the file + handleOneInput(options, input, inputFd, outputFile, outputFd, errorHolder); + if (errorHolder.hasError()) { + continue; + } + // Delete the input file if necessary + if (!options.keepSource) { + // Be sure that we are done and have written everything before we delete + if (!errorHolder.check(std::fclose(inputFd) == 0, + "Failed to close input file")) { + continue; + } + closeInputGuard.dismiss(); + if (!errorHolder.check(std::fclose(outputFd) == 0, + "Failed to close output file")) { + continue; + } + closeOutputGuard.dismiss(); + if (std::remove(input.c_str()) != 0) { + errorHolder.setError("Failed to remove input file"); + continue; + } + } + } + // Returns 1 if any of the files failed to (de)compress. + return returnCode; +} + /// Construct a `ZSTD_inBuffer` that points to the data in `buffer`. static ZSTD_inBuffer makeZstdInBuffer(const Buffer& buffer) { return ZSTD_inBuffer{buffer.data(), buffer.size(), 0}; @@ -451,12 +552,12 @@ size_t writeFile( ErrorHolder& errorHolder, WorkQueue>& outs, FILE* outputFd, - bool writeSkippableFrames) { + bool decompress) { size_t bytesWritten = 0; std::shared_ptr out; // Grab the output queue for each decompression job (in order). while (outs.pop(out) && !errorHolder.hasError()) { - if (writeSkippableFrames) { + if (!decompress) { // If we are compressing and want to write skippable frames we can't // start writing before compression is done because we need to know the // compressed size. diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h index 51d15846c..0c21d1352 100644 --- a/contrib/pzstd/Pzstd.h +++ b/contrib/pzstd/Pzstd.h @@ -28,11 +28,9 @@ namespace pzstd { * An error occurred if `errorHandler.hasError()`. * * @param options The pzstd options to use for (de)compression - * @param errorHolder Used to report errors and coordinate early shutdown - * if an error occured - * @returns The number of bytes written. + * @returns 0 upon success and non-zero on failure. */ -std::size_t pzstdMain(const Options& options, ErrorHolder& errorHolder); +int pzstdMain(const Options& options); /** * Streams input from `fd`, breaks input up into chunks, and compresses each @@ -79,16 +77,16 @@ void asyncDecompressFrames( * Streams input in from each queue in `outs` in order, and writes the data to * `outputFd`. * - * @param errorHolder Used to report errors and coordinate early exit - * @param outs A queue of output queues, one for each - * (de)compression job. - * @param outputFd The file descriptor to write to - * @param writeSkippableFrames Should we write pzstd headers? - * @returns The number of bytes written + * @param errorHolder Used to report errors and coordinate early exit + * @param outs A queue of output queues, one for each + * (de)compression job. + * @param outputFd The file descriptor to write to + * @param decompress Are we decompressing? + * @returns The number of bytes written */ std::size_t writeFile( ErrorHolder& errorHolder, WorkQueue>& outs, FILE* outputFd, - bool writeSkippableFrames); + bool decompress); } diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp index 7ff2cef74..279cbfb5e 100644 --- a/contrib/pzstd/main.cpp +++ b/contrib/pzstd/main.cpp @@ -19,16 +19,14 @@ using namespace pzstd; int main(int argc, const char** argv) { Options options; - if (!options.parse(argc, argv)) { + switch (options.parse(argc, argv)) { + case Options::Status::Failure: return 1; + case Options::Status::Message: + return 0; + default: + break; } - ErrorHolder errorHolder; - pzstdMain(options, errorHolder); - - if (errorHolder.hasError()) { - std::fprintf(stderr, "Error: %s.\n", errorHolder.getError().c_str()); - return 1; - } - return 0; + return pzstdMain(options); } diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp index b87358c04..8871dc3fb 100644 --- a/contrib/pzstd/test/OptionsTest.cpp +++ b/contrib/pzstd/test/OptionsTest.cpp @@ -8,172 +8,538 @@ */ #include "Options.h" -#include #include +#include using namespace pzstd; namespace pzstd { -bool operator==(const Options& lhs, const Options& rhs) { +bool operator==(const Options &lhs, const Options &rhs) { return lhs.numThreads == rhs.numThreads && - lhs.maxWindowLog == rhs.maxWindowLog && - lhs.compressionLevel == rhs.compressionLevel && - lhs.decompress == rhs.decompress && lhs.inputFile == rhs.inputFile && - lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite && - lhs.pzstdHeaders == rhs.pzstdHeaders; + lhs.maxWindowLog == rhs.maxWindowLog && + lhs.compressionLevel == rhs.compressionLevel && + lhs.decompress == rhs.decompress && lhs.inputFiles == rhs.inputFiles && + lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite && + lhs.keepSource == rhs.keepSource && lhs.writeMode == rhs.writeMode && + lhs.checksum == rhs.checksum && lhs.verbosity == rhs.verbosity; } + +std::ostream &operator<<(std::ostream &out, const Options &opt) { + out << "{"; + { + out << "\n\t" + << "numThreads: " << opt.numThreads; + out << ",\n\t" + << "maxWindowLog: " << opt.maxWindowLog; + out << ",\n\t" + << "compressionLevel: " << opt.compressionLevel; + out << ",\n\t" + << "decompress: " << opt.decompress; + out << ",\n\t" + << "inputFiles: {"; + { + bool first = true; + for (const auto &file : opt.inputFiles) { + if (!first) { + out << ","; + } + first = false; + out << "\n\t\t" << file; + } + } + out << "\n\t}"; + out << ",\n\t" + << "outputFile: " << opt.outputFile; + out << ",\n\t" + << "overwrite: " << opt.overwrite; + out << ",\n\t" + << "keepSource: " << opt.keepSource; + out << ",\n\t" + << "writeMode: " << static_cast(opt.writeMode); + out << ",\n\t" + << "checksum: " << opt.checksum; + out << ",\n\t" + << "verbosity: " << opt.verbosity; + } + out << "\n}"; + return out; +} +} + +namespace { +#ifdef _WIN32 +const char nullOutput[] = "nul"; +#else +const char nullOutput[] = "/dev/null"; +#endif + +const auto autoMode = Options::WriteMode::Auto; +const auto regMode = Options::WriteMode::Regular; +const auto sparseMode = Options::WriteMode::Sparse; +const auto success = Options::Status::Success; +} // anonymous namespace + +#define EXPECT_SUCCESS(...) EXPECT_EQ(Options::Status::Success, __VA_ARGS__) +#define EXPECT_FAILURE(...) EXPECT_EQ(Options::Status::Failure, __VA_ARGS__) +#define EXPECT_MESSAGE(...) EXPECT_EQ(Options::Status::Message, __VA_ARGS__) + +template +std::array makeArray(Args... args) { + return {{nullptr, args...}}; } TEST(Options, ValidInputs) { { Options options; - std::array args = { - {nullptr, "--num-threads", "5", "-o", "-", "-f"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {5, 23, 3, false, "-", "-", true, false}; + auto args = makeArray("--processes", "5", "-o", "x", "y", "-f"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {5, 23, 3, false, {"y"}, "x", + true, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = { - {nullptr, "-n", "1", "input", "-19", "-p"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 23, 19, false, "input", "input.zst", false, true}; + auto args = makeArray("-p", "1", "input", "-19"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 23, 19, false, {"input"}, "", + false, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, - "--ultra", - "-22", - "-n", - "1", - "--output", - "x", - "-d", - "x.zst", - "-f"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 0, 22, true, "x.zst", "x", true, false}; + auto args = + makeArray("--ultra", "-22", "-p", "1", "-o", "x", "-d", "x.zst", "-f"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 0, 22, true, {"x.zst"}, "x", + true, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, - "--num-threads", - "100", - "hello.zst", - "--decompress", - "--force"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {100, 23, 3, true, "hello.zst", "hello", true, false}; + auto args = makeArray("--processes", "100", "hello.zst", "--decompress", + "--force"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {100, 23, 3, true, {"hello.zst"}, "", true, + true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, "-", "-n", "1", "-c"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 23, 3, false, "-", "-", false, false}; + auto args = makeArray("x", "-dp", "1", "-c"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 23, 3, true, {"x"}, "-", + false, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, "-", "-n", "1", "--stdout"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 23, 3, false, "-", "-", false, false}; + auto args = makeArray("x", "-dp", "1", "--stdout"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 23, 3, true, {"x"}, "-", + false, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, - "-n", - "1", - "-", - "-5", - "-o", - "-", - "-u", - "-d", - "--pzstd-headers"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 0, 5, true, "-", "-", false, true}; + auto args = makeArray("-p", "1", "x", "-5", "-fo", "-", "--ultra", "-d"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 0, 5, true, {"x"}, "-", + true, true, autoMode, true, 2}; + EXPECT_EQ(expected, options); } { Options options; - std::array args = { - {nullptr, "silesia.tar", "-o", "silesia.tar.pzstd", "-n", "2"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = { - 2, 23, 3, false, "silesia.tar", "silesia.tar.pzstd", false, false}; + auto args = makeArray("silesia.tar", "-o", "silesia.tar.pzstd", "-p", "2"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {2, + 23, + 3, + false, + {"silesia.tar"}, + "silesia.tar.pzstd", + false, + true, + autoMode, + true, + 2}; + EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, "-n", "1"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-p", "1"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "-", "-n", "1"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-p", "1"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + } +} + +TEST(Options, GetOutputFile) { + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("-o-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("x", "y", "-o", nullOutput); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("x.zst", "-do", nullOutput); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("x.zst", "-d"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("x", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("xzst", "-d"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("xzst", "-doxx"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("xx", options.getOutputFile(options.inputFiles[0])); + } +} + +TEST(Options, MultipleFiles) { + { + Options options; + auto args = makeArray("x", "y", "z"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected; + expected.inputFiles = {"x", "y", "z"}; + expected.verbosity = 1; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("x", "y", "z", "-o", nullOutput); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected; + expected.inputFiles = {"x", "y", "z"}; + expected.outputFile = nullOutput; + expected.verbosity = 1; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("x", "y", "-o-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "y", "-o", "file"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-qqvd12qp4", "-f", "x", "--", "--rm", "-c"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {4, 23, 12, true, {"x", "--rm", "-c"}, + "", true, true, autoMode, true, + 0}; + EXPECT_EQ(expected, options); } } TEST(Options, NumThreads) { { Options options; - std::array args = {{nullptr, "-o", "-"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-dfo", "-"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "-n", "0", "-o", "-"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-p", "0", "-fo", "-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "-n", "-o", "-"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("-f", "-p", "-o", "-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } } TEST(Options, BadCompressionLevel) { { Options options; - std::array args = {{nullptr, "x", "-20"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-20"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "x", "-u", "-23"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "--ultra", "-23"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "--1"); // negative 1? + EXPECT_FAILURE(options.parse(args.size(), args.data())); } } TEST(Options, InvalidOption) { { Options options; - std::array args = {{nullptr, "x", "-x"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-x"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } } TEST(Options, BadOutputFile) { { Options options; - std::array args = {{nullptr, "notzst", "-d", "-n", "1"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("notzst", "-d", "-p", "1"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("", options.getOutputFile(options.inputFiles.front())); + } +} + +TEST(Options, BadOptionsWithArguments) { + { + Options options; + auto args = makeArray("x", "-pf"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-p", "10f"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-p"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-o"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-o"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, KeepSource) { + { + Options options; + auto args = makeArray("x", "--rm", "-k"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + } + { + Options options; + auto args = makeArray("x", "--rm", "--keep"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + } + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + } + { + Options options; + auto args = makeArray("x", "--rm"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(false, options.keepSource); + } +} + +TEST(Options, Verbosity) { + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(2, options.verbosity); + } + { + Options options; + auto args = makeArray("--quiet", "-qq", "x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(-1, options.verbosity); + } + { + Options options; + auto args = makeArray("x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(1, options.verbosity); + } + { + Options options; + auto args = makeArray("--", "x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(1, options.verbosity); + } + { + Options options; + auto args = makeArray("-qv", "x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(1, options.verbosity); + } + { + Options options; + auto args = makeArray("-v", "x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(3, options.verbosity); + } + { + Options options; + auto args = makeArray("-v", "x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(3, options.verbosity); + } +} + +TEST(Options, TestMode) { + { + Options options; + auto args = makeArray("x", "-t"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + EXPECT_EQ(true, options.decompress); + EXPECT_EQ(nullOutput, options.outputFile); + } + { + Options options; + auto args = makeArray("x", "--test", "--rm", "-ohello"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + EXPECT_EQ(true, options.decompress); + EXPECT_EQ(nullOutput, options.outputFile); + } +} + +TEST(Options, Checksum) { + { + Options options; + auto args = makeArray("x.zst", "--no-check", "-Cd"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.checksum); + } + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.checksum); + } + { + Options options; + auto args = makeArray("x", "--no-check", "--check"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.checksum); + } + { + Options options; + auto args = makeArray("x", "--no-check"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(false, options.checksum); + } +} + +TEST(Options, InputFiles) { + { + Options options; + auto args = makeArray("-cd"); + options.parse(args.size(), args.data()); + EXPECT_EQ(1, options.inputFiles.size()); + EXPECT_EQ("-", options.inputFiles[0]); + EXPECT_EQ("-", options.outputFile); + } + { + Options options; + auto args = makeArray(); + options.parse(args.size(), args.data()); + EXPECT_EQ(1, options.inputFiles.size()); + EXPECT_EQ("-", options.inputFiles[0]); + EXPECT_EQ("-", options.outputFile); + } + { + Options options; + auto args = makeArray("-d"); + options.parse(args.size(), args.data()); + EXPECT_EQ(1, options.inputFiles.size()); + EXPECT_EQ("-", options.inputFiles[0]); + EXPECT_EQ("-", options.outputFile); + } + { + Options options; + auto args = makeArray("x", "-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, InvalidOptions) { + { + Options options; + auto args = makeArray("-ibasdf"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("- "); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-n15"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-0", "x"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } } TEST(Options, Extras) { { Options options; - std::array args = {{nullptr, "-h"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("-h"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "-V"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("-H"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-V"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("--help"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("--version"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); } } diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp index 9d1256fa5..4075229ab 100644 --- a/contrib/pzstd/test/PzstdTest.cpp +++ b/contrib/pzstd/test/PzstdTest.cpp @@ -6,14 +6,14 @@ * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. */ -#include "datagen.h" #include "Pzstd.h" +#include "datagen.h" #include "test/RoundTrip.h" #include "utils/ScopeGuard.h" -#include #include #include +#include #include #include @@ -47,9 +47,8 @@ TEST(Pzstd, SmallSizes) { std::fprintf(stderr, "compression level: %u\n", level); }); Options options; - options.pzstdHeaders = headers; options.overwrite = true; - options.inputFile = inputFile; + options.inputFiles = {inputFile}; options.numThreads = numThreads; options.compressionLevel = level; ASSERT_TRUE(roundTrip(options)); @@ -87,9 +86,8 @@ TEST(Pzstd, LargeSizes) { std::fprintf(stderr, "compression level: %u\n", level); }); Options options; - options.pzstdHeaders = headers; options.overwrite = true; - options.inputFile = inputFile; + options.inputFiles = {inputFile}; options.numThreads = numThreads; options.compressionLevel = level; ASSERT_TRUE(roundTrip(options)); @@ -112,9 +110,8 @@ TEST(Pzstd, ExtremelyCompressible) { ASSERT_EQ(written, 10000); } Options options; - options.pzstdHeaders = false; options.overwrite = true; - options.inputFile = inputFile; + options.inputFiles = {inputFile}; options.numThreads = 1; options.compressionLevel = 1; ASSERT_TRUE(roundTrip(options)); diff --git a/contrib/pzstd/test/RoundTrip.h b/contrib/pzstd/test/RoundTrip.h index 829c95cac..8b9088459 100644 --- a/contrib/pzstd/test/RoundTrip.h +++ b/contrib/pzstd/test/RoundTrip.h @@ -55,7 +55,10 @@ inline bool check(std::string source, std::string decompressed) { } inline bool roundTrip(Options& options) { - std::string source = options.inputFile; + if (options.inputFiles.size() != 1) { + return false; + } + std::string source = options.inputFiles.front(); std::string compressedFile = std::tmpnam(nullptr); std::string decompressedFile = std::tmpnam(nullptr); auto guard = makeScopeGuard([&] { @@ -66,21 +69,15 @@ inline bool roundTrip(Options& options) { { options.outputFile = compressedFile; options.decompress = false; - ErrorHolder errorHolder; - pzstdMain(options, errorHolder); - if (errorHolder.hasError()) { - errorHolder.getError(); + if (pzstdMain(options) != 0) { return false; } } { options.decompress = true; - options.inputFile = compressedFile; + options.inputFiles.front() = compressedFile; options.outputFile = decompressedFile; - ErrorHolder errorHolder; - pzstdMain(options, errorHolder); - if (errorHolder.hasError()) { - errorHolder.getError(); + if (pzstdMain(options) != 0) { return false; } } diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h index 979c82b7a..c9c2b5b05 100644 --- a/contrib/pzstd/utils/FileSystem.h +++ b/contrib/pzstd/utils/FileSystem.h @@ -59,6 +59,22 @@ inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept { return is_regular_file(status(path, ec)); } +/// http://en.cppreference.com/w/cpp/filesystem/is_directory +inline bool is_directory(file_status status) noexcept { +#if defined(S_ISDIR) + return S_ISDIR(status.st_mode); +#elif !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR) + return (status.st_mode & S_IFMT) == S_IFDIR; +#else + static_assert(false, "NO POSIX stat() support."); +#endif +} + +/// http://en.cppreference.com/w/cpp/filesystem/is_directory +inline bool is_directory(StringPiece path, std::error_code& ec) noexcept { + return is_directory(status(path, ec)); +} + /// http://en.cppreference.com/w/cpp/filesystem/file_size inline std::uintmax_t file_size( StringPiece path,