diff --git a/build/lib/pgBackRestBuild/Config/Data.pm b/build/lib/pgBackRestBuild/Config/Data.pm index e22745cc5..8f40b6dc7 100644 --- a/build/lib/pgBackRestBuild/Config/Data.pm +++ b/build/lib/pgBackRestBuild/Config/Data.pm @@ -1234,6 +1234,7 @@ my %hConfigDefine = &CFGDEF_ALLOW_LIST => [ 'none', + 'bz2', 'gz', 'lz4', 'zst', diff --git a/doc/xml/contributing.xml b/doc/xml/contributing.xml index 626ec8d21..1723f01be 100644 --- a/doc/xml/contributing.xml +++ b/doc/xml/contributing.xml @@ -64,7 +64,7 @@ apt-get install rsync git devscripts build-essential valgrind lcov autoconf autoconf-archive libssl-dev zlib1g-dev libxml2-dev libpq-dev pkg-config libxml-checker-perl libyaml-libyaml-perl libdbd-pg-perl liblz4-dev liblz4-tool - zstd libzstd-dev + zstd libzstd-dev bzip2 libbz2-dev -y 2>&1 diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index 35763d7fb..cc329c0f6 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -141,6 +141,7 @@ The following compression types are supported: + bz2 - bzip2 compression format gz - gzip compression format lz4 - lz4 compression format (not available on all platforms) zst - Zstandard compression format (not available on all platforms) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 99745ee7d..35a05016d 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -63,6 +63,16 @@ Note that setting compress-type=zst will make new backups and archive incompatible (unrestorable) with prior versions of . + + + + + + Add bzip2 compression support. + + Note that setting compress-type=bz2 will make new backups and archive incompatible (unrestorable) with prior versions of . + + diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index 57d86a453..c5a82d638 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -767,7 +767,7 @@ apt-get install make gcc libpq-dev libssl-dev libxml2-dev pkg-config - liblz4-dev libzstd-dev + liblz4-dev libzstd-dev libbz2-dev -y 2>&1 @@ -775,7 +775,7 @@ yum install make gcc postgresql-devel openssl-devel libxml2-devel - lz4-devel libzstd-devel + lz4-devel libzstd-devel bzip2-devel -y 2>&1 diff --git a/src/Makefile.in b/src/Makefile.in index 025751222..91da3c472 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -42,6 +42,9 @@ SRCS = \ command/stanza/delete.c \ command/stanza/upgrade.c \ common/compress/helper.c \ + common/compress/bz2/common.c \ + common/compress/bz2/compress.c \ + common/compress/bz2/decompress.c \ common/compress/gz/common.c \ common/compress/gz/compress.c \ common/compress/gz/decompress.c \ diff --git a/src/build/configure.ac b/src/build/configure.ac index 5e70446fc..b3b0f6660 100644 --- a/src/build/configure.ac +++ b/src/build/configure.ac @@ -118,6 +118,11 @@ AC_CHECK_HEADER(libxml/parser.h, [], [AC_MSG_ERROR([header file is required])]) +# Check required bzip2 library +# ---------------------------------------------------------------------------------------------------------------------------------- +AC_CHECK_LIB([bz2], [BZ2_bzCompress], [], [AC_MSG_ERROR([library 'bz2' is required])]) +AC_CHECK_HEADER(bzlib.h, [], [AC_MSG_ERROR([header file is required])]) + # Check optional lz4 library # ---------------------------------------------------------------------------------------------------------------------------------- AC_CHECK_LIB( diff --git a/src/common/compress/bz2/common.c b/src/common/compress/bz2/common.c new file mode 100644 index 000000000..8efb567f6 --- /dev/null +++ b/src/common/compress/bz2/common.c @@ -0,0 +1,101 @@ +/*********************************************************************************************************************************** +BZ2 Common +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include + +#include "common/compress/bz2/common.h" +#include "common/debug.h" +#include "common/memContext.h" + +/**********************************************************************************************************************************/ +int +bz2Error(int error) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(INT, error); + FUNCTION_TEST_END(); + + if (error < 0) + { + const char *errorMsg; + const ErrorType *errorType = &FormatError; + + switch (error) + { + case BZ_SEQUENCE_ERROR: + { + errorMsg = "sequence error"; + errorType = &AssertError; + break; + } + + case BZ_PARAM_ERROR: + { + errorMsg = "parameter error"; + errorType = &AssertError; + break; + } + + case BZ_MEM_ERROR: + { + errorMsg = "memory error"; + errorType = &AssertError; + break; + } + + case BZ_DATA_ERROR: + { + errorMsg = "data error"; + errorType = &AssertError; + break; + } + + case BZ_DATA_ERROR_MAGIC: + { + errorMsg = "data error magic"; + errorType = &AssertError; + break; + } + + case BZ_IO_ERROR: + { + errorMsg = "io error"; + errorType = &AssertError; + break; + } + + case BZ_UNEXPECTED_EOF: + { + errorMsg = "unexpected eof"; + errorType = &AssertError; + break; + } + + case BZ_OUTBUFF_FULL: + { + errorMsg = "outbuff full"; + errorType = &AssertError; + break; + } + + case BZ_CONFIG_ERROR: + { + errorMsg = "config error"; + errorType = &AssertError; + break; + } + + default: + { + errorMsg = "unknown error"; + errorType = &AssertError; + } + } + + THROWP_FMT(errorType, "bz2 error: [%d] %s", error, errorMsg); + } + + FUNCTION_TEST_RETURN(error); +} diff --git a/src/common/compress/bz2/common.h b/src/common/compress/bz2/common.h new file mode 100644 index 000000000..e624369aa --- /dev/null +++ b/src/common/compress/bz2/common.h @@ -0,0 +1,21 @@ +/*********************************************************************************************************************************** +BZ2 Common + +Developed using the documentation in https://www.sourceware.org/bzip2/manual/manual.html#libprog +***********************************************************************************************************************************/ +#ifndef COMMON_COMPRESS_BZ2_COMMON_H +#define COMMON_COMPRESS_BZ2_COMMON_H + +#include + +/*********************************************************************************************************************************** +BZ2 extension +***********************************************************************************************************************************/ +#define BZ2_EXT "bz2" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +int bz2Error(int error); + +#endif diff --git a/src/common/compress/bz2/compress.c b/src/common/compress/bz2/compress.c new file mode 100644 index 000000000..9cafb8894 --- /dev/null +++ b/src/common/compress/bz2/compress.c @@ -0,0 +1,196 @@ +/*********************************************************************************************************************************** +BZ2 Compress +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include +#include + +#include "common/compress/bz2/common.h" +#include "common/compress/bz2/compress.h" +#include "common/debug.h" +#include "common/io/filter/filter.intern.h" +#include "common/log.h" +#include "common/memContext.h" +#include "common/type/object.h" + +/*********************************************************************************************************************************** +Filter type constant +***********************************************************************************************************************************/ +STRING_EXTERN(BZ2_COMPRESS_FILTER_TYPE_STR, BZ2_COMPRESS_FILTER_TYPE); + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +#define BZ2_COMPRESS_TYPE Bz2Compress +#define BZ2_COMPRESS_PREFIX bz2Compress + +typedef struct Bz2Compress +{ + MemContext *memContext; // Context to store data + bz_stream stream; // Compression stream + + bool inputSame; // Is the same input required on the next process call? + bool flushing; // Is input complete and flushing in progress? + bool done; // Is compression done? +} Bz2Compress; + +/*********************************************************************************************************************************** +Render as string for logging +***********************************************************************************************************************************/ +static String * +bz2CompressToLog(const Bz2Compress *this) +{ + return strNewFmt( + "{inputSame: %s, done: %s, flushing: %s, avail_in: %u}", cvtBoolToConstZ(this->inputSame), cvtBoolToConstZ(this->done), + cvtBoolToConstZ(this->flushing), this->stream.avail_in); +} + +#define FUNCTION_LOG_BZ2_COMPRESS_TYPE \ + Bz2Compress * +#define FUNCTION_LOG_BZ2_COMPRESS_FORMAT(value, buffer, bufferSize) \ + FUNCTION_LOG_STRING_OBJECT_FORMAT(value, bz2CompressToLog, buffer, bufferSize) + +/*********************************************************************************************************************************** +Free compression stream +***********************************************************************************************************************************/ +OBJECT_DEFINE_FREE_RESOURCE_BEGIN(BZ2_COMPRESS, LOG, logLevelTrace) +{ + BZ2_bzCompressEnd(&this->stream); +} +OBJECT_DEFINE_FREE_RESOURCE_END(LOG); + +/*********************************************************************************************************************************** +Compress data +***********************************************************************************************************************************/ +static void +bz2CompressProcess(THIS_VOID, const Buffer *uncompressed, Buffer *compressed) +{ + THIS(Bz2Compress); + + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(BZ2_COMPRESS, this); + FUNCTION_LOG_PARAM(BUFFER, uncompressed); + FUNCTION_LOG_PARAM(BUFFER, compressed); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(!this->done); + ASSERT(compressed != NULL); + ASSERT(!this->flushing || uncompressed == NULL); + ASSERT(this->flushing || (!this->inputSame || this->stream.avail_in != 0)); + + // If input is NULL then start flushing + if (uncompressed == NULL) + { + this->stream.avail_in = 0; + this->flushing = true; + } + // Else still have input data + else + { + // Is new input allowed? + if (!this->inputSame) + { + this->stream.avail_in = (unsigned int)bufUsed(uncompressed); + + // bzip2 does not accept const input buffers + this->stream.next_in = (char *)UNCONSTIFY(unsigned char *, bufPtrConst(uncompressed)); + } + } + + // Initialize compressed output buffer + this->stream.avail_out = (unsigned int)bufRemains(compressed); + this->stream.next_out = (char *)bufPtr(compressed) + bufUsed(compressed); + + // Perform compression, check for error + int result = bz2Error(BZ2_bzCompress(&this->stream, this->flushing ? BZ_FINISH : BZ_RUN)); + + // Set buffer used space + bufUsedSet(compressed, bufSize(compressed) - (size_t)this->stream.avail_out); + + // Is compression done? + if (this->flushing && result == BZ_STREAM_END) + this->done = true; + + // Can more input be provided on the next call? + this->inputSame = this->flushing ? !this->done : this->stream.avail_in != 0; + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Is compress done? +***********************************************************************************************************************************/ +static bool +bz2CompressDone(const THIS_VOID) +{ + THIS(const Bz2Compress); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BZ2_COMPRESS, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(this->done); +} + +/*********************************************************************************************************************************** +Is the same input required on the next process call? +***********************************************************************************************************************************/ +static bool +bz2CompressInputSame(const THIS_VOID) +{ + THIS(const Bz2Compress); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BZ2_COMPRESS, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(this->inputSame); +} + +/**********************************************************************************************************************************/ +IoFilter * +bz2CompressNew(int level) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(INT, level); + FUNCTION_LOG_END(); + + ASSERT(level > 0); + + IoFilter *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("Bz2Compress") + { + Bz2Compress *driver = memNew(sizeof(Bz2Compress)); + + *driver = (Bz2Compress) + { + .memContext = MEM_CONTEXT_NEW(), + .stream = {.bzalloc = NULL}, + }; + + // Initialize context + bz2Error(BZ2_bzCompressInit(&driver->stream, level, 0, 0)); + + // Set callback to ensure bz2 stream is freed + memContextCallbackSet(driver->memContext, bz2CompressFreeResource, driver); + + // Create param list + VariantList *paramList = varLstNew(); + varLstAdd(paramList, varNewInt(level)); + + // Create filter interface + this = ioFilterNewP( + BZ2_COMPRESS_FILTER_TYPE_STR, driver, paramList, .done = bz2CompressDone, .inOut = bz2CompressProcess, + .inputSame = bz2CompressInputSame); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_LOG_RETURN(IO_FILTER, this); +} diff --git a/src/common/compress/bz2/compress.h b/src/common/compress/bz2/compress.h new file mode 100644 index 000000000..e3353d852 --- /dev/null +++ b/src/common/compress/bz2/compress.h @@ -0,0 +1,22 @@ +/*********************************************************************************************************************************** +BZ2 Compress + +Compress IO to the bz2 format. +***********************************************************************************************************************************/ +#ifndef COMMON_COMPRESS_BZ2_COMPRESS_H +#define COMMON_COMPRESS_BZ2_COMPRESS_H + +#include "common/io/filter/filter.h" + +/*********************************************************************************************************************************** +Filter type constant +***********************************************************************************************************************************/ +#define BZ2_COMPRESS_FILTER_TYPE "bz2Compress" + STRING_DECLARE(BZ2_COMPRESS_FILTER_TYPE_STR); + +/*********************************************************************************************************************************** +Constructors +***********************************************************************************************************************************/ +IoFilter *bz2CompressNew(int level); + +#endif diff --git a/src/common/compress/bz2/decompress.c b/src/common/compress/bz2/decompress.c new file mode 100644 index 000000000..88a43a46f --- /dev/null +++ b/src/common/compress/bz2/decompress.c @@ -0,0 +1,177 @@ +/*********************************************************************************************************************************** +BZ2 Decompress +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include +#include + +#include "common/compress/bz2/common.h" +#include "common/compress/bz2/decompress.h" +#include "common/debug.h" +#include "common/io/filter/filter.intern.h" +#include "common/log.h" +#include "common/memContext.h" +#include "common/type/object.h" + +/*********************************************************************************************************************************** +Filter type constant +***********************************************************************************************************************************/ +STRING_EXTERN(BZ2_DECOMPRESS_FILTER_TYPE_STR, BZ2_DECOMPRESS_FILTER_TYPE); + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +#define BZ2_DECOMPRESS_TYPE Bz2Decompress +#define BZ2_DECOMPRESS_PREFIX bz2Decompress + +typedef struct Bz2Decompress +{ + MemContext *memContext; // Context to store data + bz_stream stream; // Decompression stream state + + int result; // Result of last operation + bool inputSame; // Is the same input required on the next process call? + bool done; // Is decompression done? +} Bz2Decompress; + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +static String * +bz2DecompressToLog(const Bz2Decompress *this) +{ + return strNewFmt( + "{inputSame: %s, done: %s, avail_in: %u}", cvtBoolToConstZ(this->inputSame), cvtBoolToConstZ(this->done), + this->stream.avail_in); +} + +#define FUNCTION_LOG_BZ2_DECOMPRESS_TYPE \ + Bz2Decompress * +#define FUNCTION_LOG_BZ2_DECOMPRESS_FORMAT(value, buffer, bufferSize) \ + FUNCTION_LOG_STRING_OBJECT_FORMAT(value, bz2DecompressToLog, buffer, bufferSize) + +/*********************************************************************************************************************************** +Free inflate stream +***********************************************************************************************************************************/ +OBJECT_DEFINE_FREE_RESOURCE_BEGIN(BZ2_DECOMPRESS, LOG, logLevelTrace) +{ + BZ2_bzDecompressEnd(&this->stream); +} +OBJECT_DEFINE_FREE_RESOURCE_END(LOG); + +/*********************************************************************************************************************************** +Decompress data +***********************************************************************************************************************************/ +static void +bz2DecompressProcess(THIS_VOID, const Buffer *compressed, Buffer *uncompressed) +{ + THIS(Bz2Decompress); + + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(BZ2_DECOMPRESS, this); + FUNCTION_LOG_PARAM(BUFFER, compressed); + FUNCTION_LOG_PARAM(BUFFER, uncompressed); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(uncompressed != NULL); + + // There should never be a flush because in a valid compressed stream the end of data can be determined and done will be set. + // If a flush is received it means the compressed stream terminated early, e.g. a zero-length or truncated file. + if (compressed == NULL) + THROW(FormatError, "unexpected eof in compressed data"); + + if (!this->inputSame) + { + this->stream.avail_in = (unsigned int)bufUsed(compressed); + + // bzip2 does not accept const input buffers + this->stream.next_in = (char *)UNCONSTIFY(unsigned char *, bufPtrConst(compressed)); + } + + this->stream.avail_out = (unsigned int)bufRemains(uncompressed); + this->stream.next_out = (char *)bufPtr(uncompressed) + bufUsed(uncompressed); + + this->result = bz2Error(BZ2_bzDecompress(&this->stream)); + + // Set buffer used space + bufUsedSet(uncompressed, bufSize(uncompressed) - (size_t)this->stream.avail_out); + + // Is decompression done? + this->done = this->result == BZ_STREAM_END; + + // Is the same input expected on the next call? + this->inputSame = this->done ? false : this->stream.avail_in != 0; + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Is decompress done? +***********************************************************************************************************************************/ +static bool +bz2DecompressDone(const THIS_VOID) +{ + THIS(const Bz2Decompress); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BZ2_DECOMPRESS, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(this->done); +} + +/*********************************************************************************************************************************** +Is the same input required on the next process call? +***********************************************************************************************************************************/ +static bool +bz2DecompressInputSame(const THIS_VOID) +{ + THIS(const Bz2Decompress); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BZ2_DECOMPRESS, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(this->inputSame); +} + +/**********************************************************************************************************************************/ +IoFilter * +bz2DecompressNew(void) +{ + FUNCTION_LOG_VOID(logLevelTrace); + + IoFilter *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("Bz2Decompress") + { + // Allocate state and set context + Bz2Decompress *driver = memNew(sizeof(Bz2Decompress)); + + *driver = (Bz2Decompress) + { + .memContext = MEM_CONTEXT_NEW(), + .stream = {.bzalloc = NULL}, + }; + + // Create bz2 stream + bz2Error(driver->result = BZ2_bzDecompressInit(&driver->stream, 0, 0)); + + // Set free callback to ensure bz2 context is freed + memContextCallbackSet(driver->memContext, bz2DecompressFreeResource, driver); + + // Create filter interface + this = ioFilterNewP( + BZ2_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = bz2DecompressDone, .inOut = bz2DecompressProcess, + .inputSame = bz2DecompressInputSame); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_LOG_RETURN(IO_FILTER, this); +} diff --git a/src/common/compress/bz2/decompress.h b/src/common/compress/bz2/decompress.h new file mode 100644 index 000000000..a6b558f45 --- /dev/null +++ b/src/common/compress/bz2/decompress.h @@ -0,0 +1,22 @@ +/*********************************************************************************************************************************** +BZ2 Decompress + +Decompress IO from the bz2 format. +***********************************************************************************************************************************/ +#ifndef COMMON_COMPRESS_BZ2_DECOMPRESS_H +#define COMMON_COMPRESS_BZ2_DECOMPRESS_H + +#include "common/io/filter/filter.h" + +/*********************************************************************************************************************************** +Filter type constant +***********************************************************************************************************************************/ +#define BZ2_DECOMPRESS_FILTER_TYPE "bz2Decompress" + STRING_DECLARE(BZ2_DECOMPRESS_FILTER_TYPE_STR); + +/*********************************************************************************************************************************** +Constructors +***********************************************************************************************************************************/ +IoFilter *bz2DecompressNew(void); + +#endif diff --git a/src/common/compress/helper.c b/src/common/compress/helper.c index d5842635e..d187f5c72 100644 --- a/src/common/compress/helper.c +++ b/src/common/compress/helper.c @@ -6,6 +6,9 @@ Compression Helper #include #include "common/compress/helper.h" +#include "common/compress/bz2/common.h" +#include "common/compress/bz2/compress.h" +#include "common/compress/bz2/decompress.h" #include "common/compress/gz/common.h" #include "common/compress/gz/compress.h" #include "common/compress/gz/decompress.h" @@ -25,7 +28,6 @@ Compression type constants #define COMPRESS_TYPE_NONE "none" // Constants for currently unsupported compression types -#define BZ2_EXT "bz2" #define XZ_EXT "xz" /*********************************************************************************************************************************** @@ -46,6 +48,15 @@ static const struct CompressHelperLocal .type = STRDEF(COMPRESS_TYPE_NONE), .ext = STRDEF(""), }, + { + .type = STRDEF(BZ2_EXT), + .ext = STRDEF("." BZ2_EXT), + .compressType = BZ2_COMPRESS_FILTER_TYPE, + .compressNew = bz2CompressNew, + .decompressType = BZ2_DECOMPRESS_FILTER_TYPE, + .decompressNew = bz2DecompressNew, + .levelDefault = 9, + }, { .type = STRDEF(GZ_EXT), .ext = STRDEF("." GZ_EXT), diff --git a/src/common/compress/helper.h b/src/common/compress/helper.h index 53d74ad10..8f4492af2 100644 --- a/src/common/compress/helper.h +++ b/src/common/compress/helper.h @@ -15,6 +15,7 @@ Available compression types typedef enum { compressTypeNone, // No compression + compressTypeBz2, // bzip2 compressTypeGz, // gzip compressTypeLz4, // lz4 compressTypeZst, // zstandard @@ -23,7 +24,6 @@ typedef enum // versions. In that sense this list is speculative, but these seem to be all the types that are likely to be added in the // foreseeable future. compressTypeXz, // xz/lzma - compressTypeBz2, // bzip2 } CompressType; #include diff --git a/src/config/define.auto.c b/src/config/define.auto.c index 6492864ea..713d58248 100644 --- a/src/config/define.auto.c +++ b/src/config/define.auto.c @@ -832,6 +832,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST ( "The following compression types are supported:\n" "\n" + "* bz2 - bzip2 compression format\n" "* gz - gzip compression format\n" "* lz4 - lz4 compression format (not available on all platforms)\n" "* zst - Zstandard compression format (not available on all platforms)" @@ -848,6 +849,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST ( "none", + "bz2", "gz", "lz4", "zst" diff --git a/src/configure b/src/configure index e6451a50b..95f3d9233 100755 --- a/src/configure +++ b/src/configure @@ -4018,6 +4018,64 @@ fi +# Check required bzip2 library +# ---------------------------------------------------------------------------------------------------------------------------------- +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for BZ2_bzCompress in -lbz2" >&5 +$as_echo_n "checking for BZ2_bzCompress in -lbz2... " >&6; } +if ${ac_cv_lib_bz2_BZ2_bzCompress+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lbz2 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char BZ2_bzCompress (); +int +main () +{ +return BZ2_bzCompress (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_bz2_BZ2_bzCompress=yes +else + ac_cv_lib_bz2_BZ2_bzCompress=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bz2_BZ2_bzCompress" >&5 +$as_echo "$ac_cv_lib_bz2_BZ2_bzCompress" >&6; } +if test "x$ac_cv_lib_bz2_BZ2_bzCompress" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBBZ2 1 +_ACEOF + + LIBS="-lbz2 $LIBS" + +else + as_fn_error $? "library 'bz2' is required" "$LINENO" 5 +fi + +ac_fn_c_check_header_mongrel "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" "$ac_includes_default" +if test "x$ac_cv_header_bzlib_h" = xyes; then : + +else + as_fn_error $? "header file is required" "$LINENO" 5 +fi + + + # Check optional lz4 library # ---------------------------------------------------------------------------------------------------------------------------------- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for LZ4F_isError in -llz4" >&5 @@ -5417,4 +5475,4 @@ if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} fi -# Generated from src/build/configure.ac sha1 93fbf56725f3dab1089ddd4d7768f4414bf7941c +# Generated from src/build/configure.ac sha1 a4acd8b59dcdff7e986a8454335119e955d63c91 diff --git a/test/Vagrantfile b/test/Vagrantfile index 41b4c3bcf..c3d981784 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -75,7 +75,7 @@ Vagrant.configure(2) do |config| #----------------------------------------------------------------------------------------------------------------------- echo 'Install Build Tools' && date apt-get install -y devscripts build-essential lintian git cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \ - libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev + libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev bzip2 libbz2-dev #----------------------------------------------------------------------------------------------------------------------- echo 'Install Docker' && date diff --git a/test/container.yaml b/test/container.yaml index 27688a163..b8b59e546 100644 --- a/test/container.yaml +++ b/test/container.yaml @@ -12,13 +12,9 @@ # - docker login -u pgbackrest # - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?} # ********************************************************************************************************************************** -20200504A: - co7: 9c922915380c0beb3d813c94d53f823d7008c69b - f30: 9ad36516c2403b835c24efbbcef359322a80fa1f - u18: 32426ecf8cd0e98976298f6f6b41a2b255aca692 - -20200310A: - co6: beb7b5a62ebdf209bb54494129baeadbd0c2954f - -20200124A: - u12: 0f2fcf1bc79ee35e78121773c9a2155a77cb10d5 +20200505A: + co6: c07889acc321e461263fada797941cfa947d2550 + co7: 538dc7fd9cc129e44d4ae6bacfb6f1db10309993 + f30: 3ae2c1749afa24daa1ca31abe9441d5e1e66d892 + u12: 8a88ab44aace049d7da5ca1094375ff8b9aeb7ab + u18: 0f88948969b70f300a9134482e43cda7b94cb24d diff --git a/test/define.yaml b/test/define.yaml index 895e795e4..70a9e65ee 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -262,9 +262,12 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: compress - total: 4 + total: 5 coverage: + common/compress/bz2/common: full + common/compress/bz2/compress: full + common/compress/bz2/decompress: full common/compress/gz/common: full common/compress/gz/compress: full common/compress/gz/decompress: full diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index b2e96604a..b3d754e6d 100755 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -361,7 +361,7 @@ sub containerBuild " yum -y install openssh-server openssh-clients wget sudo valgrind git \\\n" . " perl perl-Digest-SHA perl-DBD-Pg perl-YAML-LibYAML openssl \\\n" . " gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel perl-ExtUtils-Embed rpm-build \\\n" . - " zlib-devel libxml2-devel lz4-devel lz4"; + " zlib-devel libxml2-devel lz4-devel lz4 bzip2-devel bzip2"; if ($strOS eq VM_CO6) { @@ -380,7 +380,8 @@ sub containerBuild " apt-get -y install openssh-server wget sudo gcc make valgrind git \\\n" . " libdbd-pg-perl libhtml-parser-perl libssl-dev libperl-dev \\\n" . " libyaml-libyaml-perl tzdata devscripts lintian libxml-checker-perl txt2man debhelper \\\n" . - " libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev pkg-config"; + " libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev pkg-config \\\n" . + " libbz2-dev bzip2"; if ($strOS eq VM_U12) { diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 1743c1185..90d315387 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -518,7 +518,7 @@ sub run "BUILDFLAGS=${strBuildFlags}\n" . "HARNESSFLAGS=${strHarnessFlags}\n" . "TESTFLAGS=${strTestFlags}\n" . - "LDFLAGS=-lcrypto -lssl -lxml2 -lz" . + "LDFLAGS=-lcrypto -lssl -lxml2 -lz -lbz2" . (vmWithLz4($self->{oTest}->{&TEST_VM}) ? ' -llz4' : '') . (vmWithZst($self->{oTest}->{&TEST_VM}) ? ' -lzstd' : '') . (vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') . diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm index c11aa953e..396262ee2 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm @@ -95,6 +95,8 @@ use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid'; use constant NONE => 'none'; push @EXPORT, qw(NONE); +use constant BZ2 => 'bz2'; + push @EXPORT, qw(BZ2); use constant GZ => 'gz'; push @EXPORT, qw(GZ); use constant LZ4 => 'lz4'; diff --git a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm index 18fdd3b14..09cdf6e31 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm @@ -206,12 +206,12 @@ sub run foreach my $rhRun ( {vm => VM1, remote => false, s3 => true, encrypt => false, delta => true, compress => LZ4}, - {vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => GZ}, - {vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => GZ}, + {vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => BZ2}, + {vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => BZ2}, {vm => VM2, remote => true, s3 => true, encrypt => false, delta => false, compress => GZ}, {vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => ZST}, {vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => LZ4}, - {vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => LZ4}, + {vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => GZ}, {vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => ZST}, ) { diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm index af3dcae2e..c72821c5a 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm @@ -46,7 +46,7 @@ sub run {vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4, error => 0}, {vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ, error => 1}, {vm => VM2, remote => false, s3 => true, encrypt => false, compress => NONE, error => 0}, - {vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ, error => 0}, + {vm => VM2, remote => true, s3 => false, encrypt => true, compress => BZ2, error => 0}, {vm => VM3, remote => false, s3 => false, encrypt => true, compress => NONE, error => 0}, {vm => VM3, remote => true, s3 => true, encrypt => false, compress => LZ4, error => 1}, {vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST, error => 0}, diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm index a9a405bb2..2dc37b03b 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm @@ -83,14 +83,14 @@ sub run foreach my $rhRun ( - {vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4}, - {vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ}, - {vm => VM2, remote => false, s3 => true, encrypt => false, compress => GZ}, - {vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ}, + {vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ}, + {vm => VM1, remote => true, s3 => true, encrypt => true, compress => LZ4}, + {vm => VM2, remote => false, s3 => true, encrypt => false, compress => BZ2}, + {vm => VM2, remote => true, s3 => false, encrypt => true, compress => BZ2}, {vm => VM3, remote => false, s3 => false, encrypt => true, compress => LZ4}, {vm => VM3, remote => true, s3 => true, encrypt => false, compress => ZST}, {vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST}, - {vm => VM4, remote => true, s3 => false, encrypt => false, compress => LZ4}, + {vm => VM4, remote => true, s3 => false, encrypt => false, compress => GZ}, ) { # Only run tests for this vm diff --git a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm index 0f4489365..4301654de 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm @@ -42,12 +42,12 @@ sub run foreach my $rhRun ( {vm => VM1, remote => false, s3 => false, encrypt => true, compress => LZ4}, - {vm => VM1, remote => true, s3 => true, encrypt => false, compress => GZ}, - {vm => VM2, remote => false, s3 => true, encrypt => true, compress => GZ}, + {vm => VM1, remote => true, s3 => true, encrypt => false, compress => BZ2}, + {vm => VM2, remote => false, s3 => true, encrypt => true, compress => BZ2}, {vm => VM2, remote => true, s3 => false, encrypt => false, compress => GZ}, {vm => VM3, remote => false, s3 => false, encrypt => false, compress => ZST}, {vm => VM3, remote => true, s3 => true, encrypt => true, compress => LZ4}, - {vm => VM4, remote => false, s3 => true, encrypt => false, compress => LZ4}, + {vm => VM4, remote => false, s3 => true, encrypt => false, compress => GZ}, {vm => VM4, remote => true, s3 => false, encrypt => true, compress => ZST}, ) { diff --git a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm index 730f43769..f65e18077 100644 --- a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm @@ -66,7 +66,7 @@ sub run { my $strCompressType = $bHostBackup && !$bHostStandby ? - (vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : vmWithZst($self->vm()) ? ZST : GZ) : NONE; + (vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : vmWithZst($self->vm()) ? ZST : ($bS3 ? BZ2 : GZ)) : NONE; my $bRepoEncrypt = ($strCompressType ne NONE && !$bS3) ? true : false; # If compression was used then switch it for the next test that uses compression diff --git a/test/patch/debian-package.patch b/test/patch/debian-package.patch index 6c319ed78..5cf814aee 100644 --- a/test/patch/debian-package.patch +++ b/test/patch/debian-package.patch @@ -1,12 +1,13 @@ --- control +++ control -@@ -13,7 +13,8 @@ +@@ -13,7 +13,9 @@ pkg-config, txt2man, zlib1g-dev, - liblz4-dev + liblz4-dev, + libzstd-dev, ++ libbz2-dev, Standards-Version: 4.4.1 Homepage: https://www.pgbackrest.org/ Vcs-Git: https://salsa.debian.org/postgresql/pgbackrest.git diff --git a/test/src/module/common/compressTest.c b/test/src/module/common/compressTest.c index 22a1cfec5..e40bd9c99 100644 --- a/test/src/module/common/compressTest.c +++ b/test/src/module/common/compressTest.c @@ -147,19 +147,23 @@ testSuite(CompressType type, const char *decompressCmd) TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data"); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("compress a large zero input buffer into small output buffer"); + TEST_TITLE("compress a large non-zero input buffer into small output buffer"); decompressed = bufNew(1024 * 1024 - 1); - memset(bufPtr(decompressed), 0, bufSize(decompressed)); + unsigned char *c = bufPtr(decompressed); + + for (size_t i = 0; i < bufSize(decompressed); i++) + c[i] = (unsigned char)(i % 94 + 32); + bufUsedSet(decompressed, bufSize(decompressed)); TEST_ASSIGN( compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32), - "zero data - compress large in/small out buffer"); + "non-zero data - compress large in/small out buffer"); TEST_RESULT_BOOL( bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true, - "zero data - decompress large in/small out buffer"); + "non-zero data - decompress large in/small out buffer"); } /*********************************************************************************************************************************** @@ -203,6 +207,49 @@ testRun(void) TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: true, done: true, availIn: 0}", "format object"); } + // ***************************************************************************************************************************** + if (testBegin("bz2")) + { + // Run standard test suite + testSuite(compressTypeBz2, "bzip2 -dc"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("bz2Error()"); + + TEST_RESULT_INT(bz2Error(BZ_OK), BZ_OK, "check ok"); + TEST_RESULT_INT(bz2Error(BZ_RUN_OK), BZ_RUN_OK, "check run ok"); + TEST_RESULT_INT(bz2Error(BZ_FLUSH_OK), BZ_FLUSH_OK, "check flush ok"); + TEST_RESULT_INT(bz2Error(BZ_FINISH_OK), BZ_FINISH_OK, "check finish ok"); + TEST_RESULT_INT(bz2Error(BZ_STREAM_END), BZ_STREAM_END, "check stream end"); + TEST_ERROR(bz2Error(BZ_SEQUENCE_ERROR), AssertError, "bz2 error: [-1] sequence error"); + TEST_ERROR(bz2Error(BZ_PARAM_ERROR), AssertError, "bz2 error: [-2] parameter error"); + TEST_ERROR(bz2Error(BZ_MEM_ERROR), AssertError, "bz2 error: [-3] memory error"); + TEST_ERROR(bz2Error(BZ_DATA_ERROR), AssertError, "bz2 error: [-4] data error"); + TEST_ERROR(bz2Error(BZ_DATA_ERROR_MAGIC), AssertError, "bz2 error: [-5] data error magic"); + TEST_ERROR(bz2Error(BZ_IO_ERROR), AssertError, "bz2 error: [-6] io error"); + TEST_ERROR(bz2Error(BZ_UNEXPECTED_EOF), AssertError, "bz2 error: [-7] unexpected eof"); + TEST_ERROR(bz2Error(BZ_OUTBUFF_FULL), AssertError, "bz2 error: [-8] outbuff full"); + TEST_ERROR(bz2Error(BZ_CONFIG_ERROR), AssertError, "bz2 error: [-9] config error"); + TEST_ERROR(bz2Error(-999), AssertError, "bz2 error: [-999] unknown error"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("bz2DecompressToLog() and bz2CompressToLog()"); + + Bz2Compress *compress = (Bz2Compress *)ioFilterDriver(bz2CompressNew(1)); + + compress->stream.avail_in = 999; + + TEST_RESULT_STR_Z( + bz2CompressToLog(compress), "{inputSame: false, done: false, flushing: false, avail_in: 999}", "format object"); + + Bz2Decompress *decompress = (Bz2Decompress *)ioFilterDriver(bz2DecompressNew()); + + decompress->inputSame = true; + decompress->done = true; + + TEST_RESULT_STR_Z(bz2DecompressToLog(decompress), "{inputSame: true, done: true, avail_in: 0}", "format object"); + } + // ***************************************************************************************************************************** if (testBegin("lz4")) { diff --git a/test/travis.pl b/test/travis.pl index 561bd1441..1a004e683 100755 --- a/test/travis.pl +++ b/test/travis.pl @@ -186,7 +186,7 @@ eval # Extra packages required when testing without containers if ($strVm eq VM_NONE) { - $strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev"; + $strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev"; } # Else packages needed for integration tests on containers else
Note that setting compress-type=zst will make new backups and archive incompatible (unrestorable) with prior versions of .
Add bzip2 compression support.
Note that setting compress-type=bz2 will make new backups and archive incompatible (unrestorable) with prior versions of .