diff --git a/build/lib/pgBackRestBuild/Config/Data.pm b/build/lib/pgBackRestBuild/Config/Data.pm index a6786c334..ab4800235 100644 --- a/build/lib/pgBackRestBuild/Config/Data.pm +++ b/build/lib/pgBackRestBuild/Config/Data.pm @@ -169,6 +169,8 @@ use constant CFGOPT_DB_TIMEOUT => 'db-timeo push @EXPORT, qw(CFGOPT_DB_TIMEOUT); use constant CFGOPT_COMPRESS => 'compress'; push @EXPORT, qw(CFGOPT_COMPRESS); +use constant CFGOPT_COMPRESS_TYPE => 'compress-type'; + push @EXPORT, qw(CFGOPT_COMPRESS_TYPE); use constant CFGOPT_COMPRESS_LEVEL => 'compress-level'; push @EXPORT, qw(CFGOPT_COMPRESS_LEVEL); use constant CFGOPT_COMPRESS_LEVEL_NETWORK => 'compress-level-network'; @@ -1152,6 +1154,7 @@ my %hConfigDefine = }, }, + # Option is deprecated and should not be referenced outside of cfgLoadUpdateOption(). &CFGOPT_COMPRESS => { &CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL, @@ -1164,11 +1167,24 @@ my %hConfigDefine = } }, + &CFGOPT_COMPRESS_TYPE => + { + &CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL, + &CFGDEF_TYPE => CFGDEF_TYPE_STRING, + &CFGDEF_DEFAULT => 'gz', + &CFGDEF_ALLOW_LIST => + [ + 'none', + 'gz', + ], + &CFGDEF_COMMAND => CFGOPT_COMPRESS, + }, + &CFGOPT_COMPRESS_LEVEL => { &CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL, &CFGDEF_TYPE => CFGDEF_TYPE_INTEGER, - &CFGDEF_DEFAULT => 6, + &CFGDEF_REQUIRED => false, &CFGDEF_ALLOW_RANGE => [CFGDEF_DEFAULT_COMPRESS_LEVEL_MIN, CFGDEF_DEFAULT_COMPRESS_LEVEL_MAX], &CFGDEF_COMMAND => CFGOPT_COMPRESS, }, diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index 714c7d31e..4266fe7d3 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -126,18 +126,32 @@ - Use gzip file compression. + Use file compression. - Backup files are compatible with command-line gzip tools. + Backup files are compatible with command-line compression tools. + + This option is now deprecated. The compress-type option should be used instead. + + n + + + + + File compression type. + + The following compression types are supported: + n - Compression level for stored files. + File compression level. - Sets the zlib level to be used for file compression when compress=y. + Sets the level to be used for file compression when compress-type does not equal none or compress=y (deprecated). 0-9 9 @@ -145,9 +159,9 @@ - Compression level for network transfer when compress=n. + Network compression level. - Sets the zlib level to be used for protocol compression when compress=n and the database cluster is not on the same host as the repository. Protocol compression is used to reduce network traffic but can be disabled by setting compress-level-network=0. When compress=y the compress-level-network setting is ignored and compress-level is used instead so that the file is only compressed once. SSH compression is always disabled. + Sets the network compression level when compress-type=none and the command is not run on the same host as the repository. Compression is used to reduce network traffic but can be disabled by setting compress-level-network=0. When compress-type does not equal none the compress-level-network setting is ignored and compress-level is used instead so that the file is only compressed once. SSH compression is always disabled. 0-9 1 diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 4331a6886..c9e6bef41 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -27,6 +27,14 @@ + + + + + +

Add infrastructure for multiple compression type support.

+ + diff --git a/lib/pgBackRest/LibCAuto.pm b/lib/pgBackRest/LibCAuto.pm index 46ccc8da6..40f2abb29 100644 --- a/lib/pgBackRest/LibCAuto.pm +++ b/lib/pgBackRest/LibCAuto.pm @@ -11,6 +11,9 @@ sub libcAutoConstant { return { + CFGOPTVAL_COMPRESS_TYPE_NONE => 'none', + CFGOPTVAL_COMPRESS_TYPE_GZ => 'gz', + CFGOPTVAL_INFO_OUTPUT_TEXT => 'text', CFGOPTVAL_INFO_OUTPUT_JSON => 'json', @@ -78,6 +81,8 @@ sub libcAutoExportTag { config => [ + 'CFGOPTVAL_COMPRESS_TYPE_NONE', + 'CFGOPTVAL_COMPRESS_TYPE_GZ', 'CFGOPTVAL_INFO_OUTPUT_TEXT', 'CFGOPTVAL_INFO_OUTPUT_JSON', 'CFGOPTVAL_LS_OUTPUT_TEXT', @@ -139,6 +144,7 @@ sub libcAutoExportTag 'CFGOPT_COMPRESS', 'CFGOPT_COMPRESS_LEVEL', 'CFGOPT_COMPRESS_LEVEL_NETWORK', + 'CFGOPT_COMPRESS_TYPE', 'CFGOPT_CONFIG', 'CFGOPT_CONFIG_INCLUDE_PATH', 'CFGOPT_CONFIG_PATH', diff --git a/lib/pgBackRest/Manifest.pm b/lib/pgBackRest/Manifest.pm index 7bbd97400..d48fe0a35 100644 --- a/lib/pgBackRest/Manifest.pm +++ b/lib/pgBackRest/Manifest.pm @@ -108,6 +108,8 @@ use constant MANIFEST_KEY_CHECKSUM_PAGE => 'option-' push @EXPORT, qw(MANIFEST_KEY_CHECKSUM_PAGE); use constant MANIFEST_KEY_COMPRESS => 'option-' . cfgOptionName(CFGOPT_COMPRESS); push @EXPORT, qw(MANIFEST_KEY_COMPRESS); +use constant MANIFEST_KEY_COMPRESS_TYPE => 'option-' . cfgOptionName(CFGOPT_COMPRESS_TYPE); + push @EXPORT, qw(MANIFEST_KEY_COMPRESS_TYPE); use constant MANIFEST_KEY_COMPRESS_LEVEL => 'option-' . cfgOptionName(CFGOPT_COMPRESS_LEVEL); push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL); use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK => 'option-' . cfgOptionName(CFGOPT_COMPRESS_LEVEL_NETWORK); diff --git a/libc/Makefile.PL b/libc/Makefile.PL index fae5f175c..60cd546fc 100644 --- a/libc/Makefile.PL +++ b/libc/Makefile.PL @@ -46,6 +46,7 @@ my @stryCFile = 'common/compress/gz/common.c', 'common/compress/gz/compress.c', 'common/compress/gz/decompress.c', + 'common/compress/helper.c', 'common/crypto/cipherBlock.c', 'common/crypto/common.c', 'common/crypto/hash.c', diff --git a/src/Makefile.in b/src/Makefile.in index 329acf8c9..00d6134d4 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -75,6 +75,7 @@ SRCS = \ command/stanza/delete.c \ command/stanza/upgrade.c \ command/storage/list.c \ + common/compress/helper.c \ common/compress/gz/common.c \ common/compress/gz/compress.c \ common/compress/gz/decompress.c \ @@ -212,46 +213,46 @@ clean: #################################################################################################################################### # Compile rules #################################################################################################################################### -command/archive/common.o: command/archive/common.c build.auto.h command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/archive/common.o: command/archive/common.c build.auto.h command/archive/common.h common/assert.h common/compress/helper.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/common.c -o command/archive/common.o -command/archive/get/file.o: command/archive/get/file.c build.auto.h command/archive/common.h command/archive/get/file.h command/control/common.h common/assert.h common/compress/gz/common.h common/compress/gz/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoPg.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/archive/get/file.o: command/archive/get/file.c build.auto.h command/archive/common.h command/archive/get/file.h command/control/common.h common/assert.h common/compress/helper.h common/crypto/cipherBlock.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoPg.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/get/file.c -o command/archive/get/file.o -command/archive/get/get.o: command/archive/get/get.c build.auto.h command/archive/common.h command/archive/get/file.h command/archive/get/protocol.h command/command.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/archive/get/get.o: command/archive/get/get.c build.auto.h command/archive/common.h command/archive/get/file.h command/archive/get/protocol.h command/command.h common/assert.h common/compress/helper.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/get/get.c -o command/archive/get/get.o command/archive/get/protocol.o: command/archive/get/protocol.c build.auto.h command/archive/get/file.h command/archive/get/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/get/protocol.c -o command/archive/get/protocol.o -command/archive/push/file.o: command/archive/push/file.c build.auto.h command/archive/common.h command/archive/push/file.h command/control/common.h common/assert.h common/compress/gz/common.h common/compress/gz/compress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/archive/push/file.o: command/archive/push/file.c build.auto.h command/archive/common.h command/archive/push/file.h command/control/common.h common/assert.h common/compress/helper.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/push/file.c -o command/archive/push/file.o -command/archive/push/protocol.o: command/archive/push/protocol.c build.auto.h command/archive/push/file.h command/archive/push/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/archive/push/protocol.o: command/archive/push/protocol.c build.auto.h command/archive/push/file.h command/archive/push/protocol.h common/assert.h common/compress/helper.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/push/protocol.c -o command/archive/push/protocol.o -command/archive/push/push.o: command/archive/push/push.c build.auto.h command/archive/common.h command/archive/push/file.h command/archive/push/protocol.h command/command.h command/control/common.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h info/info.h info/infoArchive.h info/infoPg.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/archive/push/push.o: command/archive/push/push.c build.auto.h command/archive/common.h command/archive/push/file.h command/archive/push/protocol.h command/command.h command/control/common.h common/assert.h common/compress/helper.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h info/info.h info/infoArchive.h info/infoPg.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/push/push.c -o command/archive/push/push.o -command/backup/backup.o: command/backup/backup.c build.auto.h command/archive/common.h command/backup/backup.h command/backup/common.h command/backup/file.h command/backup/protocol.h command/check/common.h command/control/common.h command/stanza/common.h common/assert.h common/compress/gz/common.h common/compress/gz/compress.h common/compress/gz/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h db/helper.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/client.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h +command/backup/backup.o: command/backup/backup.c build.auto.h command/archive/common.h command/backup/backup.h command/backup/common.h command/backup/file.h command/backup/protocol.h command/check/common.h command/control/common.h command/stanza/common.h common/assert.h common/compress/helper.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h db/helper.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/client.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/backup.c -o command/backup/backup.o command/backup/common.o: command/backup/common.c build.auto.h command/backup/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/string.h common/type/stringz.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/common.c -o command/backup/common.o -command/backup/file.o: command/backup/file.c build.auto.h command/backup/file.h command/backup/pageChecksum.h common/assert.h common/compress/gz/common.h common/compress/gz/compress.h common/compress/gz/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/backup/file.o: command/backup/file.c build.auto.h command/backup/file.h command/backup/pageChecksum.h common/assert.h common/compress/helper.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/file.c -o command/backup/file.o command/backup/pageChecksum.o: command/backup/pageChecksum.c build.auto.h command/backup/pageChecksum.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h postgres/interface.h postgres/interface/static.auto.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/pageChecksum.c -o command/backup/pageChecksum.o -command/backup/protocol.o: command/backup/protocol.c build.auto.h command/backup/file.h command/backup/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/backup/protocol.o: command/backup/protocol.c build.auto.h command/backup/file.h command/backup/protocol.h common/assert.h common/compress/helper.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/protocol.c -o command/backup/protocol.o -command/check/check.o: command/check/check.c build.auto.h command/archive/common.h command/check/check.h command/check/common.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h db/helper.h info/info.h info/infoArchive.h info/infoPg.h postgres/client.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/check/check.o: command/check/check.c build.auto.h command/archive/common.h command/check/check.h command/check/common.h common/assert.h common/compress/helper.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h db/helper.h info/info.h info/infoArchive.h info/infoPg.h postgres/client.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/check/check.c -o command/check/check.o -command/check/common.o: command/check/common.c build.auto.h command/backup/common.h command/check/common.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h db/helper.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/client.h postgres/interface.h protocol/client.h protocol/command.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h +command/check/common.o: command/check/common.c build.auto.h command/backup/common.h command/check/common.h common/assert.h common/compress/helper.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h db/helper.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/client.h postgres/interface.h protocol/client.h protocol/command.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/check/common.c -o command/check/common.o command/command.o: command/command.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/tls/client.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h @@ -266,13 +267,13 @@ command/control/start.o: command/control/start.c build.auto.h command/control/co command/control/stop.o: command/control/stop.c build.auto.h command/control/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/control/stop.c -o command/control/stop.o -command/expire/expire.o: command/expire/expire.c build.auto.h command/archive/common.h command/backup/common.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/expire/expire.o: command/expire/expire.c build.auto.h command/archive/common.h command/backup/common.h common/assert.h common/compress/helper.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/expire/expire.c -o command/expire/expire.o command/help/help.o: command/help/help.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/help/help.c -o command/help/help.o -command/info/info.o: command/info/info.c build.auto.h command/archive/common.h command/backup/common.h command/info/info.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/info/info.o: command/info/info.c build.auto.h command/archive/common.h command/backup/common.h command/info/info.h common/assert.h common/compress/helper.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/info/info.c -o command/info/info.o command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h command/backup/protocol.h command/restore/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h @@ -281,25 +282,25 @@ command/local/local.o: command/local/local.c build.auto.h command/archive/get/pr command/remote/remote.o: command/remote/remote.c build.auto.h command/control/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h db/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h storage/remote/protocol.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/remote/remote.c -o command/remote/remote.o -command/restore/file.o: command/restore/file.c build.auto.h command/restore/file.h common/assert.h common/compress/gz/common.h common/compress/gz/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/restore/file.o: command/restore/file.c build.auto.h command/restore/file.h common/assert.h common/compress/helper.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/restore/file.c -o command/restore/file.o -command/restore/protocol.o: command/restore/protocol.c build.auto.h command/restore/file.h command/restore/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/restore/protocol.o: command/restore/protocol.c build.auto.h command/restore/file.h command/restore/protocol.h common/assert.h common/compress/helper.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/restore/protocol.c -o command/restore/protocol.o -command/restore/restore.o: command/restore/restore.c build.auto.h command/backup/common.h command/restore/protocol.h command/restore/restore.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/user.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h info/info.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h storage/write.intern.h version.h +command/restore/restore.o: command/restore/restore.c build.auto.h command/backup/common.h command/restore/protocol.h command/restore/restore.h common/assert.h common/compress/helper.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/user.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h info/info.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h storage/write.intern.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/restore/restore.c -o command/restore/restore.o command/stanza/common.o: command/stanza/common.c build.auto.h command/check/common.h common/assert.h common/crypto/common.h common/debug.h common/encode.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h db/helper.h info/info.h info/infoPg.h postgres/client.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/stanza/common.c -o command/stanza/common.o -command/stanza/create.o: command/stanza/create.c build.auto.h command/backup/common.h command/check/common.h command/control/common.h command/stanza/common.h command/stanza/create.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/client.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/stanza/create.o: command/stanza/create.c build.auto.h command/backup/common.h command/check/common.h command/control/common.h command/stanza/common.h command/stanza/create.h common/assert.h common/compress/helper.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/client.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/stanza/create.c -o command/stanza/create.o -command/stanza/delete.o: command/stanza/delete.c build.auto.h command/backup/common.h command/control/common.h command/stanza/delete.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/stanza/delete.o: command/stanza/delete.c build.auto.h command/backup/common.h command/control/common.h command/stanza/delete.h common/assert.h common/compress/helper.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/stanza/delete.c -o command/stanza/delete.o -command/stanza/upgrade.o: command/stanza/upgrade.c build.auto.h command/backup/common.h command/check/common.h command/control/common.h command/stanza/common.h command/stanza/upgrade.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/client.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/stanza/upgrade.o: command/stanza/upgrade.c build.auto.h command/backup/common.h command/check/common.h command/control/common.h command/stanza/common.h command/stanza/upgrade.h common/assert.h common/compress/helper.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/client.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/stanza/upgrade.c -o command/stanza/upgrade.o command/storage/list.o: command/storage/list.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h @@ -314,6 +315,9 @@ common/compress/gz/compress.o: common/compress/gz/compress.c build.auto.h common common/compress/gz/decompress.o: common/compress/gz/decompress.c build.auto.h common/assert.h common/compress/gz/common.h common/compress/gz/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/compress/gz/decompress.c -o common/compress/gz/decompress.o +common/compress/helper.o: common/compress/helper.c build.auto.h common/assert.h common/compress/gz/common.h common/compress/gz/compress.h common/compress/gz/decompress.h common/compress/helper.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringz.h common/type/variant.h common/type/variantList.h version.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/compress/helper.c -o common/compress/helper.o + common/crypto/cipherBlock.o: common/crypto/cipherBlock.c build.auto.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/io/filter/group.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/crypto/cipherBlock.c -o common/crypto/cipherBlock.o @@ -467,7 +471,7 @@ config/define.o: config/define.c build.auto.h common/assert.h common/debug.h com config/exec.o: config/exec.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c config/exec.c -o config/exec.o -config/load.o: config/load.c build.auto.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/io.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h +config/load.o: config/load.c build.auto.h command/command.h common/assert.h common/compress/helper.h common/compress/helper.intern.h common/debug.h common/error.auto.h common/error.h common/io/io.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h config/parse.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c config/load.c -o config/load.o config/parse.o: config/parse.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/parse.auto.c config/parse.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h @@ -491,13 +495,13 @@ info/info.o: info/info.c build.auto.h common/assert.h common/crypto/hash.h commo info/infoArchive.o: info/infoArchive.c build.auto.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h info/info.h info/infoArchive.h info/infoPg.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/infoArchive.c -o info/infoArchive.o -info/infoBackup.o: info/infoBackup.c build.auto.h command/backup/common.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h info/info.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h +info/infoBackup.o: info/infoBackup.c build.auto.h command/backup/common.h common/assert.h common/compress/helper.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h info/info.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/infoBackup.c -o info/infoBackup.o info/infoPg.o: info/infoPg.c build.auto.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h info/info.h info/infoPg.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/infoPg.c -o info/infoPg.o -info/manifest.o: info/manifest.c build.auto.h command/backup/common.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/mcv.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h info/info.h info/manifest.h postgres/interface.h postgres/version.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h +info/manifest.o: info/manifest.c build.auto.h command/backup/common.h common/assert.h common/compress/helper.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/mcv.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h info/info.h info/manifest.h postgres/interface.h postgres/version.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/manifest.c -o info/manifest.o main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/backup/backup.h command/check/check.h command/command.h command/control/start.h command/control/stop.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/restore/restore.h command/stanza/create.h command/stanza/delete.h command/stanza/upgrade.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h @@ -584,16 +588,16 @@ storage/posix/write.o: storage/posix/write.c build.auto.h common/assert.h common storage/read.o: storage/read.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringz.h common/type/variant.h common/type/variantList.h storage/read.h storage/read.intern.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/read.c -o storage/read.o -storage/remote/protocol.o: storage/remote/protocol.c build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/gz/compress.h common/compress/gz/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/sink.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h +storage/remote/protocol.o: storage/remote/protocol.c build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/helper.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/sink.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/protocol.c -o storage/remote/protocol.o -storage/remote/read.o: storage/remote/read.c build.auto.h common/assert.h common/compress/gz/compress.h common/compress/gz/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/read.h storage/remote/storage.h storage/remote/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h +storage/remote/read.o: storage/remote/read.c build.auto.h common/assert.h common/compress/helper.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/read.h storage/remote/storage.h storage/remote/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/read.c -o storage/remote/read.o storage/remote/storage.o: storage/remote/storage.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/read.h storage/remote/storage.h storage/remote/storage.intern.h storage/remote/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/storage.c -o storage/remote/storage.o -storage/remote/write.o: storage/remote/write.c build.auto.h common/assert.h common/compress/gz/compress.h common/compress/gz/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/storage.h storage/remote/storage.intern.h storage/remote/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h +storage/remote/write.o: storage/remote/write.c build.auto.h common/assert.h common/compress/helper.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/storage.h storage/remote/storage.intern.h storage/remote/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/write.c -o storage/remote/write.o storage/s3/read.o: storage/s3/read.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h storage/info.h storage/read.h storage/read.intern.h storage/s3/read.h storage/s3/storage.h storage/s3/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h diff --git a/src/command/archive/common.c b/src/command/archive/common.c index afb2451ed..2555100e2 100644 --- a/src/command/archive/common.c +++ b/src/command/archive/common.c @@ -415,7 +415,7 @@ walSegmentFind(const Storage *storage, const String *archiveId, const String *wa // Get a list of all WAL segments that match StringList *list = storageListP( storage, strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(strSubN(walSegment, 0, 16))), - .expression = strNewFmt("^%s%s-[0-f]{40}(\\.gz){0,1}$", strPtr(strSubN(walSegment, 0, 24)), + .expression = strNewFmt("^%s%s-[0-f]{40}" COMPRESS_TYPE_REGEXP "{0,1}$", strPtr(strSubN(walSegment, 0, 24)), walIsPartial(walSegment) ? WAL_SEGMENT_PARTIAL_EXT : ""), .nullOnMissing = true); // If there are results diff --git a/src/command/archive/common.h b/src/command/archive/common.h index a1588f6e3..ba8f7a5c7 100644 --- a/src/command/archive/common.h +++ b/src/command/archive/common.h @@ -17,6 +17,7 @@ typedef enum archiveModeGet, } ArchiveMode; +#include "common/compress/helper.h" #include "common/type/stringList.h" #include "storage/storage.h" @@ -52,7 +53,7 @@ WAL segment constants // WAL segment directory/file #define WAL_SEGMENT_DIR_REGEXP "^[0-F]{16}$" STRING_DECLARE(WAL_SEGMENT_DIR_REGEXP_STR); -#define WAL_SEGMENT_FILE_REGEXP "^[0-F]{24}-[0-f]{40}(\\.gz){0,1}$" +#define WAL_SEGMENT_FILE_REGEXP "^[0-F]{24}-[0-f]{40}" COMPRESS_TYPE_REGEXP "{0,1}$" STRING_DECLARE(WAL_SEGMENT_FILE_REGEXP_STR); /*********************************************************************************************************************************** diff --git a/src/command/archive/get/file.c b/src/command/archive/get/file.c index b403c5b94..2b1e71f17 100644 --- a/src/command/archive/get/file.c +++ b/src/command/archive/get/file.c @@ -6,8 +6,7 @@ Archive Get File #include "command/archive/get/file.h" #include "command/archive/common.h" #include "command/control/common.h" -#include "common/compress/gz/common.h" -#include "common/compress/gz/decompress.h" +#include "common/compress/helper.h" #include "common/crypto/cipherBlock.h" #include "common/debug.h" #include "common/io/filter/group.h" @@ -160,9 +159,11 @@ archiveGetFile( } // If file is compressed then add the decompression filter - if (strEndsWithZ(archiveGetCheckResult.archiveFileActual, "." GZ_EXT)) + CompressType compressType = compressTypeFromName(archiveGetCheckResult.archiveFileActual); + + if (compressType != compressTypeNone) { - ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(destination)), gzDecompressNew()); + ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(destination)), decompressFilter(compressType)); compressible = false; } diff --git a/src/command/archive/push/file.c b/src/command/archive/push/file.c index b12f54e9b..60a73f499 100644 --- a/src/command/archive/push/file.c +++ b/src/command/archive/push/file.c @@ -6,8 +6,6 @@ Archive Push File #include "command/archive/push/file.h" #include "command/archive/common.h" #include "command/control/common.h" -#include "common/compress/gz/common.h" -#include "common/compress/gz/compress.h" #include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" @@ -24,7 +22,7 @@ Copy a file from the source to the archive String * archivePushFile( const String *walSource, const String *archiveId, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile, - CipherType cipherType, const String *cipherPass, bool compress, int compressLevel) + CipherType cipherType, const String *cipherPass, CompressType compressType, int compressLevel) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, walSource); @@ -34,7 +32,7 @@ archivePushFile( FUNCTION_LOG_PARAM(STRING, archiveFile); FUNCTION_LOG_PARAM(ENUM, cipherType); FUNCTION_TEST_PARAM(STRING, cipherPass); - FUNCTION_LOG_PARAM(BOOL, compress); + FUNCTION_LOG_PARAM(ENUM, compressType); FUNCTION_LOG_PARAM(INT, compressLevel); FUNCTION_LOG_END(); @@ -114,10 +112,10 @@ archivePushFile( bool compressible = true; // If the file will be compressed then add compression filter - if (isSegment && compress) + if (isSegment && compressType != compressTypeNone) { - strCat(archiveDestination, "." GZ_EXT); - ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(source)), gzCompressNew(compressLevel)); + compressExtCat(archiveDestination, compressType); + ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(source)), compressFilter(compressType, compressLevel)); compressible = false; } diff --git a/src/command/archive/push/file.h b/src/command/archive/push/file.h index 1d529f539..cba37ac67 100644 --- a/src/command/archive/push/file.h +++ b/src/command/archive/push/file.h @@ -4,6 +4,7 @@ Archive Push File #ifndef COMMAND_ARCHIVE_PUSH_FILE_H #define COMMAND_ARCHIVE_PUSH_FILE_H +#include "common/compress/helper.h" #include "common/crypto/common.h" #include "common/type/string.h" #include "storage/storage.h" @@ -13,6 +14,6 @@ Functions ***********************************************************************************************************************************/ String *archivePushFile( const String *walSource, const String *archiveId, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile, - CipherType cipherType, const String *cipherPass, bool compress, int compressLevel); + CipherType cipherType, const String *cipherPass, CompressType compressType, int compressLevel); #endif diff --git a/src/command/archive/push/protocol.c b/src/command/archive/push/protocol.c index 530affbd3..12beb14e0 100644 --- a/src/command/archive/push/protocol.c +++ b/src/command/archive/push/protocol.c @@ -48,7 +48,7 @@ archivePushProtocol(const String *command, const VariantList *paramList, Protoco varStr(varLstGet(paramList, 0)), varStr(varLstGet(paramList, 1)), varUIntForce(varLstGet(paramList, 2)), varUInt64(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)), (CipherType)varUIntForce(varLstGet(paramList, 5)), varStr(varLstGet(paramList, 6)), - varBool(varLstGet(paramList, 7)), varIntForce(varLstGet(paramList, 8))))); + (CompressType)varUIntForce(varLstGet(paramList, 7)), varIntForce(varLstGet(paramList, 8))))); } else found = false; diff --git a/src/command/archive/push/push.c b/src/command/archive/push/push.c index 8e36c5b98..0a4541929 100644 --- a/src/command/archive/push/push.c +++ b/src/command/archive/push/push.c @@ -11,6 +11,7 @@ Archive Push Command #include "command/archive/push/protocol.h" #include "command/command.h" #include "command/control/common.h" +#include "common/compress/helper.h" #include "common/debug.h" #include "common/log.h" #include "common/memContext.h" @@ -359,7 +360,7 @@ cmdArchivePush(void) String *warning = archivePushFile( walFile, archiveInfo.archiveId, archiveInfo.pgVersion, archiveInfo.pgSystemId, archiveFile, cipherType(cfgOptionStr(cfgOptRepoCipherType)), archiveInfo.archiveCipherPass, - cfgOptionBool(cfgOptCompress), cfgOptionInt(cfgOptCompressLevel)); + compressTypeEnum(cfgOptionStr(cfgOptCompressType)), cfgOptionInt(cfgOptCompressLevel)); // If a warning was returned then log it if (warning != NULL) @@ -384,7 +385,7 @@ typedef struct ArchivePushAsyncData const StringList *walFileList; // List of wal files to process unsigned int walFileIdx; // Current index in the list to be processed CipherType cipherType; // Cipher type - bool compress; // Compress wal files + CompressType compressType; // Type of compression for WAL segments int compressLevel; // Compression level for wal files ArchivePushCheckResult archiveInfo; // Archive info } ArchivePushAsyncData; @@ -415,7 +416,7 @@ static ProtocolParallelJob *archivePushAsyncCallback(void *data, unsigned int cl protocolCommandParamAdd(command, VARSTR(walFile)); protocolCommandParamAdd(command, VARUINT(jobData->cipherType)); protocolCommandParamAdd(command, VARSTR(jobData->archiveInfo.archiveCipherPass)); - protocolCommandParamAdd(command, VARBOOL(jobData->compress)); + protocolCommandParamAdd(command, VARUINT(jobData->compressType)); protocolCommandParamAdd(command, VARINT(jobData->compressLevel)); FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(walFile), command)); @@ -445,7 +446,7 @@ cmdArchivePushAsync(void) ArchivePushAsyncData jobData = { .walPath = strLstGet(commandParam, 0), - .compress = cfgOptionBool(cfgOptCompress), + .compressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)), .compressLevel = cfgOptionInt(cfgOptCompressLevel), }; diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c index 1ed138462..8a729d6f7 100644 --- a/src/command/backup/backup.c +++ b/src/command/backup/backup.c @@ -17,9 +17,7 @@ Backup Command #include "command/check/common.h" #include "command/stanza/common.h" #include "common/crypto/cipherBlock.h" -#include "common/compress/gz/common.h" -#include "common/compress/gz/compress.h" -#include "common/compress/gz/decompress.h" +#include "common/compress/helper.h" #include "common/debug.h" #include "common/io/filter/size.h" #include "common/log.h" @@ -124,8 +122,9 @@ backupLabelCreate(BackupType type, const String *backupLabelPrior, time_t timest storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/" BACKUP_PATH_HISTORY "/%s", strPtr(strLstGet(historyYearList, 0))), .expression = strNewFmt( - "%s\\.manifest\\." GZ_EXT "$", - strPtr(backupRegExpP(.full = true, .differential = true, .incremental = true, .noAnchorEnd = true)))), + "%s\\.manifest\\.%s$", + strPtr(backupRegExpP(.full = true, .differential = true, .incremental = true, .noAnchorEnd = true)), + strPtr(compressTypeStr(compressTypeGz)))), sortOrderDesc); if (strLstSize(historyList) > 0) @@ -397,14 +396,27 @@ backupBuildIncrPrior(const InfoBackup *infoBackup) "last backup label = %s, version = %s", strPtr(manifestData(result)->backupLabel), strPtr(manifestData(result)->backrestVersion)); - // Warn if compress option changed - if (cfgOptionBool(cfgOptCompress) != manifestPriorData->backupOptionCompress) + // Warn if compress-type option changed + if (compressTypeEnum(cfgOptionStr(cfgOptCompressType)) != manifestPriorData->backupOptionCompressType) { LOG_WARN_FMT( - "%s backup cannot alter compress option to '%s', reset to value in %s", - strPtr(cfgOptionStr(cfgOptType)), cvtBoolToConstZ(cfgOptionBool(cfgOptCompress)), - strPtr(backupLabelPrior)); - cfgOptionSet(cfgOptCompress, cfgSourceParam, VARBOOL(manifestPriorData->backupOptionCompress)); + "%s backup cannot alter " CFGOPT_COMPRESS_TYPE " option to '%s', reset to value in %s", + strPtr(cfgOptionStr(cfgOptType)), + strPtr(compressTypeStr(compressTypeEnum(cfgOptionStr(cfgOptCompressType)))), strPtr(backupLabelPrior)); + + // Set the compression type back to whatever was in the prior backup. This is not strictly needed since we + // could store compression type on a per file basis, but it seems simplest and safest for now. + cfgOptionSet( + cfgOptCompressType, cfgSourceParam, VARSTR(compressTypeStr(manifestPriorData->backupOptionCompressType))); + + // There's a small chance that the prior manifest is old enough that backupOptionCompressLevel was not recorded. + // There's an even smaller chance that the user will also alter compression-type in this this scenario right + // after upgrading to a newer version. Because we judge this combination of events to be nearly impossible just + // assert here so no test coverage is needed. + CHECK(manifestPriorData->backupOptionCompressLevel != NULL); + + // Set the compression level back to whatever was in the prior backup + cfgOptionSet(cfgOptCompressLevel, cfgSourceParam, manifestPriorData->backupOptionCompressLevel); } // Warn if hardlink option changed ??? Doesn't seem like this is needed? Hardlinks are always to a directory that @@ -502,7 +514,7 @@ typedef struct BackupResumeData { Manifest *manifest; // New manifest const Manifest *manifestResume; // Resumed manifest - const bool compressed; // Is the backup compressed? + const CompressType compressType; // Backup compression type const bool delta; // Is this a delta backup? const String *backupPath; // Path to the current level of the backup being cleaned const String *manifestParentName; // Parent manifest name used to construct manifest name @@ -573,9 +585,11 @@ void backupResumeCallback(void *data, const StorageInfo *info) // ------------------------------------------------------------------------------------------------------------------------- case storageTypeFile: { - // If the backup is compressed then strip off the extension before doing the lookup - if (resumeData->compressed) - manifestName = strSubN(manifestName, 0, strSize(manifestName) - sizeof(GZ_EXT)); + // If the file is compressed then strip off the extension before doing the lookup + CompressType fileCompressType = compressTypeFromName(manifestName); + + if (fileCompressType != compressTypeNone) + manifestName = compressExtStrip(manifestName, fileCompressType); // Find the file in both manifests const ManifestFile *file = manifestFileFindDefault(resumeData->manifest, manifestName, NULL); @@ -584,7 +598,9 @@ void backupResumeCallback(void *data, const StorageInfo *info) // Check if the file can be resumed or must be removed const char *removeReason = NULL; - if (file == NULL) + if (fileCompressType != resumeData->compressType) + removeReason = "mismatched compression type"; + else if (file == NULL) removeReason = "missing in manifest"; else if (file->reference != NULL) removeReason = "reference in manifest"; @@ -713,12 +729,12 @@ backupResumeFind(const Manifest *manifest, const String *cipherPassBackup) strPtr(manifestData(manifest)->backupLabelPrior) : ""); } // Check compression. Compression can't be changed between backups so resume won't work either. - else if (manifestResumeData->backupOptionCompress != cfgOptionBool(cfgOptCompress)) + else if (manifestResumeData->backupOptionCompressType != compressTypeEnum(cfgOptionStr(cfgOptCompressType))) { reason = strNewFmt( "new compression '%s' does not match resumable compression '%s'", - cvtBoolToConstZ(cfgOptionBool(cfgOptCompress)), - cvtBoolToConstZ(manifestResumeData->backupOptionCompress)); + strPtr(compressTypeStr(compressTypeEnum(cfgOptionStr(cfgOptCompressType)))), + strPtr(compressTypeStr(manifestResumeData->backupOptionCompressType))); } else usable = true; @@ -788,7 +804,7 @@ backupResume(Manifest *manifest, const String *cipherPassBackup) { .manifest = manifest, .manifestResume = manifestResume, - .compressed = cfgOptionBool(cfgOptCompress), + .compressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)), .delta = cfgOptionBool(cfgOptDelta), .backupPath = strNewFmt(STORAGE_REPO_BACKUP "/%s", strPtr(manifestData(manifest)->backupLabel)), }; @@ -918,13 +934,13 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti { // Create file const String *manifestName = strNewFmt(MANIFEST_TARGET_PGDATA "/%s", strPtr(name)); - bool compress = cfgOptionBool(cfgOptCompress); + CompressType compressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)); StorageWrite *write = storageNewWriteP( storageRepoWrite(), strNewFmt( STORAGE_REPO_BACKUP "/%s/%s%s", strPtr(manifestData(manifest)->backupLabel), strPtr(manifestName), - compress ? "." GZ_EXT : ""), + strPtr(compressExtStr(compressType))), .compressible = true); IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(write)); @@ -933,8 +949,11 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti ioFilterGroupAdd(filterGroup, cryptoHashNew(HASH_TYPE_SHA1_STR)); // Add compression - if (compress) - ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), gzCompressNew((int)cfgOptionUInt(cfgOptCompressLevel))); + if (compressType != compressTypeNone) + { + ioFilterGroupAdd( + ioWriteFilterGroup(storageWriteIo(write)), compressFilter(compressType, cfgOptionInt(cfgOptCompressLevel))); + } // Add encryption filter if required cipherBlockFilterGroupAdd( @@ -1418,8 +1437,8 @@ typedef struct BackupJobData const String *const backupLabel; // Backup label (defines the backup path) const bool backupStandby; // Backup from standby const String *const cipherSubPass; // Passphrase used to encrypt files in the backup - const bool compress; // Is the backup compressed? - const unsigned int compressLevel; // Compress level if backup is compressed + const CompressType compressType; // Backup compression type + const int compressLevel; // Compress level if backup is compressed const bool delta; // Is this a checksum delta backup? const uint64_t lsnStart; // Starting lsn for the backup @@ -1469,8 +1488,8 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx protocolCommandParamAdd(command, VARUINT64(jobData->lsnStart)); protocolCommandParamAdd(command, VARSTR(file->name)); protocolCommandParamAdd(command, VARBOOL(file->reference != NULL)); - protocolCommandParamAdd(command, VARBOOL(jobData->compress)); - protocolCommandParamAdd(command, VARUINT(jobData->compressLevel)); + protocolCommandParamAdd(command, VARUINT(jobData->compressType)); + protocolCommandParamAdd(command, VARINT(jobData->compressLevel)); protocolCommandParamAdd(command, VARSTR(jobData->backupLabel)); protocolCommandParamAdd(command, VARBOOL(jobData->delta)); protocolCommandParamAdd(command, VARSTR(jobData->cipherSubPass)); @@ -1561,8 +1580,8 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart { .backupLabel = backupLabel, .backupStandby = backupStandby, - .compress = cfgOptionBool(cfgOptCompress), - .compressLevel = cfgOptionUInt(cfgOptCompressLevel), + .compressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)), + .compressLevel = cfgOptionInt(cfgOptCompressLevel), .cipherSubPass = manifestCipherSubPass(manifest), .delta = cfgOptionBool(cfgOptDelta), .lsnStart = cfgOptionBool(cfgOptOnline) ? pgLsnFromStr(lsnStart) : 0xFFFFFFFFFFFFFFFF, @@ -1646,7 +1665,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart manifestFileRemove(manifest, strLstGet(fileRemove, fileRemoveIdx)); // Log references or create hardlinks for all files - const char *const compressExt = jobData.compress ? "." GZ_EXT : ""; + const char *const compressExt = strPtr(compressExtStr(jobData.compressType)); for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { @@ -1751,8 +1770,9 @@ backupArchiveCheckCopy(Manifest *manifest, unsigned int walSegmentSize, const St if (cfgOptionBool(cfgOptArchiveCopy)) { - // Is the archive file compressed? - bool archiveCompressed = strEndsWithZ(archiveFile, "." GZ_EXT); + // Get compression type of the WAL segment and backup + CompressType archiveCompressType = compressTypeFromName(archiveFile); + CompressType backupCompressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)); // Open the archive file StorageRead *read = storageNewReadP( @@ -1764,13 +1784,14 @@ backupArchiveCheckCopy(Manifest *manifest, unsigned int walSegmentSize, const St filterGroup, cipherType(cfgOptionStr(cfgOptRepoCipherType)), cipherModeDecrypt, infoArchiveCipherPass(infoArchive)); - // Compress or decompress if archive and backup do not have the same compression settings - if (archiveCompressed != cfgOptionBool(cfgOptCompress)) + // Compress/decompress if archive and backup do not have the same compression settings + if (archiveCompressType != backupCompressType) { - if (archiveCompressed) - ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzDecompressNew()); - else - ioFilterGroupAdd(filterGroup, gzCompressNew(cfgOptionInt(cfgOptCompressLevel))); + if (archiveCompressType != compressTypeNone) + ioFilterGroupAdd(filterGroup, decompressFilter(archiveCompressType)); + + if (backupCompressType != compressTypeNone) + ioFilterGroupAdd(filterGroup, compressFilter(backupCompressType, cfgOptionInt(cfgOptCompressLevel))); } // Encrypt with backup key if encrypted @@ -1791,7 +1812,7 @@ backupArchiveCheckCopy(Manifest *manifest, unsigned int walSegmentSize, const St storageRepoWrite(), strNewFmt( STORAGE_REPO_BACKUP "/%s/%s%s", strPtr(manifestData(manifest)->backupLabel), strPtr(manifestName), - cfgOptionBool(cfgOptCompress) ? "." GZ_EXT : ""))); + strPtr(compressExtStr(compressTypeEnum(cfgOptionStr(cfgOptCompressType))))))); // Add to manifest ManifestFile file = @@ -1861,10 +1882,10 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest) StorageWrite *manifestWrite = storageNewWriteP( storageRepoWrite(), strNewFmt( - STORAGE_REPO_BACKUP "/" BACKUP_PATH_HISTORY "/%s/%s.manifest." GZ_EXT, strPtr(strSubN(backupLabel, 0, 4)), - strPtr(backupLabel))); + STORAGE_REPO_BACKUP "/" BACKUP_PATH_HISTORY "/%s/%s.manifest%s", strPtr(strSubN(backupLabel, 0, 4)), + strPtr(backupLabel), strPtr(compressExtStr(compressTypeGz)))); - ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), gzCompressNew(9)); + ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), compressFilter(compressTypeGz, 9)); cipherBlockFilterGroupAdd( ioWriteFilterGroup(storageWriteIo(manifestWrite)), cipherType(cfgOptionStr(cfgOptRepoCipherType)), cipherModeEncrypt, @@ -1949,7 +1970,8 @@ cmdBackup(void) strLstNewVarLst(cfgOptionLst(cfgOptExclude)), backupStartResult.tablespaceList); // Validate the manifest using the copy start time - manifestBuildValidate(manifest, cfgOptionBool(cfgOptDelta), backupTime(backupData, true), cfgOptionBool(cfgOptCompress)); + manifestBuildValidate( + manifest, cfgOptionBool(cfgOptDelta), backupTime(backupData, true), compressTypeEnum(cfgOptionStr(cfgOptCompressType))); // Build an incremental backup if type is not full (manifestPrior will be freed in this call) if (!backupBuildIncr(infoBackup, manifest, manifestPrior, backupStartResult.walSegmentName)) diff --git a/src/command/backup/file.c b/src/command/backup/file.c index dab92a4a6..bd07a6a6f 100644 --- a/src/command/backup/file.c +++ b/src/command/backup/file.c @@ -7,9 +7,6 @@ Backup File #include "command/backup/file.h" #include "command/backup/pageChecksum.h" -#include "common/compress/gz/common.h" -#include "common/compress/gz/compress.h" -#include "common/compress/gz/decompress.h" #include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" @@ -42,8 +39,8 @@ Copy a file from the PostgreSQL data directory to the repository BackupFileResult backupFile( const String *pgFile, bool pgFileIgnoreMissing, uint64_t pgFileSize, const String *pgFileChecksum, bool pgFileChecksumPage, - uint64_t pgFileChecksumPageLsnLimit, const String *repoFile, bool repoFileHasReference, bool repoFileCompress, - unsigned int repoFileCompressLevel, const String *backupLabel, bool delta, CipherType cipherType, const String *cipherPass) + uint64_t pgFileChecksumPageLsnLimit, const String *repoFile, bool repoFileHasReference, CompressType repoFileCompressType, + int repoFileCompressLevel, const String *backupLabel, bool delta, CipherType cipherType, const String *cipherPass) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, pgFile); // Database file to copy to the repo @@ -54,8 +51,8 @@ backupFile( FUNCTION_LOG_PARAM(UINT64, pgFileChecksumPageLsnLimit); // Upper LSN limit to which page checksums must be valid FUNCTION_LOG_PARAM(STRING, repoFile); // Destination in the repo to copy the pg file FUNCTION_LOG_PARAM(BOOL, repoFileHasReference); // Does the repo file exist in a prior backup in the set? - FUNCTION_LOG_PARAM(BOOL, repoFileCompress); // Compress destination file - FUNCTION_LOG_PARAM(UINT, repoFileCompressLevel); // Compression level for destination file + FUNCTION_LOG_PARAM(ENUM, repoFileCompressType); // Compress type for repo file + FUNCTION_LOG_PARAM(INT, repoFileCompressLevel); // Compression level for repo file FUNCTION_LOG_PARAM(STRING, backupLabel); // Label of current backup FUNCTION_LOG_PARAM(BOOL, delta); // Is the delta option on? FUNCTION_LOG_PARAM(ENUM, cipherType); // Encryption type @@ -74,7 +71,7 @@ backupFile( { // Generate complete repo path and add compression extension if needed const String *repoPathFile = strNewFmt( - STORAGE_REPO_BACKUP "/%s/%s%s", strPtr(backupLabel), strPtr(repoFile), repoFileCompress ? "." GZ_EXT : ""); + STORAGE_REPO_BACKUP "/%s/%s%s", strPtr(backupLabel), strPtr(repoFile), strPtr(compressExtStr(repoFileCompressType))); // If checksum is defined then the file needs to be checked. If delta option then check the DB and possibly the repo, else // just check the repo. @@ -148,8 +145,9 @@ backupFile( ioReadFilterGroup(read), cipherBlockNew(cipherModeDecrypt, cipherType, BUFSTR(cipherPass), NULL)); } - if (repoFileCompress) - ioFilterGroupAdd(ioReadFilterGroup(read), gzDecompressNew()); + // Decompress the file if compressed + if (repoFileCompressType != compressTypeNone) + ioFilterGroupAdd(ioReadFilterGroup(read), decompressFilter(repoFileCompressType)); ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR)); ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew()); @@ -190,7 +188,7 @@ backupFile( if (result.backupCopyResult == backupCopyResultCopy || result.backupCopyResult == backupCopyResultReCopy) { // Is the file compressible during the copy? - bool compressible = !repoFileCompress && cipherType == cipherTypeNone; + bool compressible = repoFileCompressType == compressTypeNone && cipherType == cipherTypeNone; // Setup pg file for read StorageRead *read = storageNewReadP( @@ -207,8 +205,11 @@ backupFile( } // Add compression - if (repoFileCompress) - ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzCompressNew((int)repoFileCompressLevel)); + if (repoFileCompressType != compressTypeNone) + { + ioFilterGroupAdd( + ioReadFilterGroup(storageReadIo(read)), compressFilter(repoFileCompressType, repoFileCompressLevel)); + } // If there is a cipher then add the encrypt filter if (cipherType != cipherTypeNone) diff --git a/src/command/backup/file.h b/src/command/backup/file.h index a6e7c9a81..0cdcc0a92 100644 --- a/src/command/backup/file.h +++ b/src/command/backup/file.h @@ -4,6 +4,7 @@ Backup File #ifndef COMMAND_BACKUP_FILE_H #define COMMAND_BACKUP_FILE_H +#include "common/compress/helper.h" #include "common/crypto/common.h" #include "common/type/keyValue.h" @@ -33,8 +34,8 @@ typedef struct BackupFileResult BackupFileResult backupFile( const String *pgFile, bool pgFileIgnoreMissing, uint64_t pgFileSize, const String *pgFileChecksum, bool pgFileChecksumPage, - uint64_t pgFileChecksumPageLsnLimit, const String *repoFile, bool repoFileHasReference, bool repoFileCompress, - unsigned int repoFileCompressLevel, const String *backupLabel, bool delta, CipherType cipherType, const String *cipherPass); + uint64_t pgFileChecksumPageLsnLimit, const String *repoFile, bool repoFileHasReference, CompressType repoFileCompressType, + int repoFileCompressLevel, const String *backupLabel, bool delta, CipherType cipherType, const String *cipherPass); /*********************************************************************************************************************************** Macros for function logging diff --git a/src/command/backup/protocol.c b/src/command/backup/protocol.c index 7631056c5..c8638a466 100644 --- a/src/command/backup/protocol.c +++ b/src/command/backup/protocol.c @@ -43,8 +43,8 @@ backupProtocol(const String *command, const VariantList *paramList, ProtocolServ varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)), varStr(varLstGet(paramList, 3)), varBool(varLstGet(paramList, 4)), varUInt64(varLstGet(paramList, 5)), varStr(varLstGet(paramList, 6)), - varBool(varLstGet(paramList, 7)), varBool(varLstGet(paramList, 8)), varUIntForce(varLstGet(paramList, 9)), - varStr(varLstGet(paramList, 10)), varBool(varLstGet(paramList, 11)), + varBool(varLstGet(paramList, 7)), (CompressType)varUIntForce(varLstGet(paramList, 8)), + varIntForce(varLstGet(paramList, 9)), varStr(varLstGet(paramList, 10)), varBool(varLstGet(paramList, 11)), varStr(varLstGet(paramList, 12)) == NULL ? cipherTypeNone : cipherTypeAes256Cbc, varStr(varLstGet(paramList, 12))); // Return backup result diff --git a/src/command/restore/file.c b/src/command/restore/file.c index b7df1ebd8..c5f882d41 100644 --- a/src/command/restore/file.c +++ b/src/command/restore/file.c @@ -8,8 +8,6 @@ Restore File #include #include "command/restore/file.h" -#include "common/compress/gz/common.h" -#include "common/compress/gz/decompress.h" #include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" @@ -25,7 +23,7 @@ Copy a file from the backup to the specified destination ***********************************************************************************************************************************/ bool restoreFile( - const String *repoFile, const String *repoFileReference, bool repoFileCompressed, const String *pgFile, + const String *repoFile, const String *repoFileReference, CompressType repoFileCompressType, const String *pgFile, const String *pgFileChecksum, bool pgFileZero, uint64_t pgFileSize, time_t pgFileModified, mode_t pgFileMode, const String *pgFileUser, const String *pgFileGroup, time_t copyTimeBegin, bool delta, bool deltaForce, const String *cipherPass) @@ -33,7 +31,7 @@ restoreFile( FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, repoFile); FUNCTION_LOG_PARAM(STRING, repoFileReference); - FUNCTION_LOG_PARAM(BOOL, repoFileCompressed); + FUNCTION_LOG_PARAM(ENUM, repoFileCompressType); FUNCTION_LOG_PARAM(STRING, pgFile); FUNCTION_LOG_PARAM(STRING, pgFileChecksum); FUNCTION_LOG_PARAM(BOOL, pgFileZero); @@ -153,9 +151,9 @@ restoreFile( } // Add decompression filter - if (repoFileCompressed) + if (repoFileCompressType != compressTypeNone) { - ioFilterGroupAdd(filterGroup, gzDecompressNew()); + ioFilterGroupAdd(filterGroup, decompressFilter(repoFileCompressType)); compressible = false; } @@ -171,7 +169,7 @@ restoreFile( storageRepo(), strNewFmt( STORAGE_REPO_BACKUP "/%s/%s%s", strPtr(repoFileReference), strPtr(repoFile), - repoFileCompressed ? "." GZ_EXT : ""), + strPtr(compressExtStr(repoFileCompressType))), .compressible = compressible), pgFileWrite); diff --git a/src/command/restore/file.h b/src/command/restore/file.h index fa532c0a2..ff90ff7ea 100644 --- a/src/command/restore/file.h +++ b/src/command/restore/file.h @@ -4,6 +4,7 @@ Restore File #ifndef COMMAND_RESTORE_FILE_H #define COMMAND_RESTORE_FILE_H +#include "common/compress/helper.h" #include "common/crypto/common.h" #include "common/type/string.h" #include "storage/storage.h" @@ -12,7 +13,7 @@ Restore File Functions ***********************************************************************************************************************************/ bool restoreFile( - const String *repoFile, const String *repoFileReference, bool repoFileCompressed, const String *pgFile, + const String *repoFile, const String *repoFileReference, CompressType repoFileCompressType, const String *pgFile, const String *pgFileChecksum, bool pgFileZero, uint64_t pgFileSize, time_t pgFileModified, mode_t pgFileMode, const String *pgFileUser, const String *pgFileGroup, time_t copyTimeBegin, bool delta, bool deltaForce, const String *cipherPass); diff --git a/src/command/restore/protocol.c b/src/command/restore/protocol.c index 4d716d98c..ee0246829 100644 --- a/src/command/restore/protocol.c +++ b/src/command/restore/protocol.c @@ -45,13 +45,13 @@ restoreProtocol(const String *command, const VariantList *paramList, ProtocolSer server, VARBOOL( restoreFile( - varStr(varLstGet(paramList, 0)), varStr(varLstGet(paramList, 1)), varBoolForce(varLstGet(paramList, 2)), - varStr(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)), varBoolForce(varLstGet(paramList, 5)), - varUInt64(varLstGet(paramList, 6)), (time_t)varInt64Force(varLstGet(paramList, 7)), - cvtZToUIntBase(strPtr(varStr(varLstGet(paramList, 8))), 8), varStr(varLstGet(paramList, 9)), - varStr(varLstGet(paramList, 10)), (time_t)varInt64Force(varLstGet(paramList, 11)), - varBoolForce(varLstGet(paramList, 12)), varBoolForce(varLstGet(paramList, 13)), - varStr(varLstGet(paramList, 14))))); + varStr(varLstGet(paramList, 0)), varStr(varLstGet(paramList, 1)), + (CompressType)varUIntForce(varLstGet(paramList, 2)), varStr(varLstGet(paramList, 3)), + varStr(varLstGet(paramList, 4)), varBoolForce(varLstGet(paramList, 5)), varUInt64(varLstGet(paramList, 6)), + (time_t)varInt64Force(varLstGet(paramList, 7)), cvtZToUIntBase(strPtr(varStr(varLstGet(paramList, 8))), 8), + varStr(varLstGet(paramList, 9)), varStr(varLstGet(paramList, 10)), + (time_t)varInt64Force(varLstGet(paramList, 11)), varBoolForce(varLstGet(paramList, 12)), + varBoolForce(varLstGet(paramList, 13)), varStr(varLstGet(paramList, 14))))); } else found = false; diff --git a/src/command/restore/restore.c b/src/command/restore/restore.c index 87b5a1cd5..51d656776 100644 --- a/src/command/restore/restore.c +++ b/src/command/restore/restore.c @@ -1962,7 +1962,7 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId protocolCommandParamAdd( command, file->reference != NULL ? VARSTR(file->reference) : VARSTR(manifestData(jobData->manifest)->backupLabel)); - protocolCommandParamAdd(command, VARBOOL(manifestData(jobData->manifest)->backupOptionCompress)); + protocolCommandParamAdd(command, VARUINT(manifestData(jobData->manifest)->backupOptionCompressType)); protocolCommandParamAdd(command, VARSTR(restoreFilePgPath(jobData->manifest, file->name))); protocolCommandParamAdd(command, VARSTRZ(file->checksumSha1)); protocolCommandParamAdd(command, VARBOOL(restoreFileZeroed(file->name, jobData->zeroExp))); diff --git a/src/common/compress/gz/compress.c b/src/common/compress/gz/compress.c index 942e487a3..4f124bcf2 100644 --- a/src/common/compress/gz/compress.c +++ b/src/common/compress/gz/compress.c @@ -199,9 +199,3 @@ gzCompressNew(int level) FUNCTION_LOG_RETURN(IO_FILTER, this); } - -IoFilter * -gzCompressNewVar(const VariantList *paramList) -{ - return gzCompressNew(varIntForce(varLstGet(paramList, 0))); -} diff --git a/src/common/compress/gz/compress.h b/src/common/compress/gz/compress.h index 963e0e151..163ed47ed 100644 --- a/src/common/compress/gz/compress.h +++ b/src/common/compress/gz/compress.h @@ -18,6 +18,5 @@ Filter type constant Constructor ***********************************************************************************************************************************/ IoFilter *gzCompressNew(int level); -IoFilter *gzCompressNewVar(const VariantList *paramList); #endif diff --git a/src/common/compress/helper.c b/src/common/compress/helper.c new file mode 100644 index 000000000..dfcc3087d --- /dev/null +++ b/src/common/compress/helper.c @@ -0,0 +1,275 @@ +/*********************************************************************************************************************************** +Compression Helper +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include + +#include "common/compress/helper.h" +#include "common/compress/gz/common.h" +#include "common/compress/gz/compress.h" +#include "common/compress/gz/decompress.h" +#include "common/debug.h" +#include "common/log.h" +#include "version.h" + +/*********************************************************************************************************************************** +Compression type constants +***********************************************************************************************************************************/ +#define COMPRESS_TYPE_NONE "none" + +// Constants for currently unsupported compression types +#define LZ4_EXT "lz4" +#define ZST_EXT "zst" +#define BZ2_EXT "bz2" +#define XZ_EXT "xz" + +/*********************************************************************************************************************************** +Configuration for supported and future compression types +***********************************************************************************************************************************/ +static const struct CompressHelperLocal +{ + const String *const type; // Compress type -- must be extension without period prefixed + const String *const ext; // File extension with period prefixed + const char *compressType; // Type of the compression filter + IoFilter *(*compressNew)(int); // Function to create new compression filter + const char *decompressType; // Type of the decompression filter + IoFilter *(*decompressNew)(void); // Function to create new decompression filter + int levelDefault; // Default compression level +} compressHelperLocal[] = +{ + { + .type = STRDEF(COMPRESS_TYPE_NONE), + .ext = STRDEF(""), + }, + { + .type = STRDEF(GZ_EXT), + .ext = STRDEF("." GZ_EXT), + .compressType = GZ_COMPRESS_FILTER_TYPE, + .compressNew = gzCompressNew, + .decompressType = GZ_DECOMPRESS_FILTER_TYPE, + .decompressNew = gzDecompressNew, + .levelDefault = 6, + }, + { + .type = STRDEF(LZ4_EXT), + .ext = STRDEF("." LZ4_EXT), + }, + { + .type = STRDEF(ZST_EXT), + .ext = STRDEF("." ZST_EXT), + }, + { + .type = STRDEF(XZ_EXT), + .ext = STRDEF("." XZ_EXT), + }, + { + .type = STRDEF(BZ2_EXT), + .ext = STRDEF("." BZ2_EXT), + }, +}; + +#define COMPRESS_LIST_SIZE \ + (sizeof(compressHelperLocal) / sizeof(struct CompressHelperLocal)) + +/**********************************************************************************************************************************/ +CompressType +compressTypeEnum(const String *type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, type); + FUNCTION_TEST_END(); + + ASSERT(type != NULL); + + CompressType result = compressTypeNone; + + for (; result < COMPRESS_LIST_SIZE; result++) + { + if (strEq(type, compressHelperLocal[result].type)) + break; + } + + if (result == COMPRESS_LIST_SIZE) + THROW_FMT(AssertError, "invalid compression type '%s'", strPtr(type)); + + FUNCTION_TEST_RETURN(result); +} + +/**********************************************************************************************************************************/ +void +compressTypePresent(CompressType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_END(); + + ASSERT(type < COMPRESS_LIST_SIZE); + + if (type != compressTypeNone && compressHelperLocal[type].compressNew == NULL) + { + THROW_FMT( + OptionInvalidValueError, PROJECT_NAME " not compiled with %s support", + strPtr(compressHelperLocal[type].type)); + } + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +const String * +compressTypeStr(CompressType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_END(); + + ASSERT(type < COMPRESS_LIST_SIZE); + + FUNCTION_TEST_RETURN(compressHelperLocal[type].type); +} + +/**********************************************************************************************************************************/ +CompressType +compressTypeFromName(const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + CompressType result = compressTypeNone + 1; + + for (; result < COMPRESS_LIST_SIZE; result++) + { + if (strEndsWith(name, compressHelperLocal[result].ext)) + break; + } + + if (result == COMPRESS_LIST_SIZE) + result = compressTypeNone; + + FUNCTION_TEST_RETURN(result); +} + +/**********************************************************************************************************************************/ +int +compressLevelDefault(CompressType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_END(); + + ASSERT(type < COMPRESS_LIST_SIZE); + compressTypePresent(type); + + FUNCTION_TEST_RETURN(compressHelperLocal[type].levelDefault); +} + +/**********************************************************************************************************************************/ +IoFilter * +compressFilter(CompressType type, int level) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_PARAM(INT, level); + FUNCTION_TEST_END(); + + ASSERT(type < COMPRESS_LIST_SIZE); + ASSERT(type != compressTypeNone); + compressTypePresent(type); + + FUNCTION_TEST_RETURN(compressHelperLocal[type].compressNew(level)); +} + +/**********************************************************************************************************************************/ +IoFilter * +compressFilterVar(const String *filterType, const VariantList *filterParamList) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(STRING, filterType); + FUNCTION_LOG_PARAM(VARIANT_LIST, filterParamList); + FUNCTION_LOG_END(); + + ASSERT(filterType != NULL); + + IoFilter *result = NULL; + + for (CompressType compressIdx = compressTypeNone + 1; compressIdx < COMPRESS_LIST_SIZE; compressIdx++) + { + const struct CompressHelperLocal *compress = &compressHelperLocal[compressIdx]; + + if (compress->compressType != NULL && strEqZ(filterType, compress->compressType)) + { + result = compress->compressNew(varIntForce(varLstGet(filterParamList, 0))); + break; + } + else if (compress->decompressType != NULL && strEqZ(filterType, compress->decompressType)) + { + result = compress->decompressNew(); + break; + } + } + + FUNCTION_LOG_RETURN(IO_FILTER, result); +} + +/**********************************************************************************************************************************/ +IoFilter * +decompressFilter(CompressType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_END(); + + ASSERT(type < COMPRESS_LIST_SIZE); + ASSERT(type != compressTypeNone); + compressTypePresent(type); + + FUNCTION_TEST_RETURN(compressHelperLocal[type].decompressNew()); +} + +/**********************************************************************************************************************************/ +const String * +compressExtStr(CompressType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_END(); + + ASSERT(type < COMPRESS_LIST_SIZE); + + FUNCTION_TEST_RETURN(compressHelperLocal[type].ext); +} + +/**********************************************************************************************************************************/ +void +compressExtCat(String *file, CompressType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, file); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_END(); + + ASSERT(file != NULL); + + strCat(file, strPtr(compressExtStr(type))); + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +String * +compressExtStrip(const String *file, CompressType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, file); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_END(); + + ASSERT(file != NULL); + + if (!strEndsWith(file, compressExtStr(type))) + THROW_FMT(FormatError, "'%s' must have '%s' extension", strPtr(file), strPtr(compressExtStr(type))); + + FUNCTION_TEST_RETURN(strSubN(file, 0, strSize(file) - strSize(compressExtStr(type)))); +} diff --git a/src/common/compress/helper.h b/src/common/compress/helper.h new file mode 100644 index 000000000..ee8e66dea --- /dev/null +++ b/src/common/compress/helper.h @@ -0,0 +1,74 @@ +/*********************************************************************************************************************************** +Compression Helper + +Abstract compression types so that the calling code does not need to worry about the individual characteristics of the compression +type. All compress calls should use this module rather than the individual filters, even if a specific compression type is required. +***********************************************************************************************************************************/ +#ifndef COMMON_COMPRESS_HELPER_H +#define COMMON_COMPRESS_HELPER_H + +#include + +/*********************************************************************************************************************************** +Available compression types +***********************************************************************************************************************************/ +typedef enum +{ + compressTypeNone, // No compression + compressTypeGz, // gzip + compressTypeLz4, // lz4 + + // These types have not been implemented but are included here so older versions can identify compression types added by future + // versions. In that sense this list is speculative, but these seem to be all the types that are likely to be added in the + // foreseeable future. + compressTypeZst, // zstandard + compressTypeXz, // xz/lzma + compressTypeBz2, // bzip2 +} CompressType; + +#include +#include + +/*********************************************************************************************************************************** +Compression types as a regexp. In the future this regexp will be generated automatically at build time but we want to wait until the +build code is migrated to C to do that. For now just define it statically since we don't expect it to change very often. If a +supported type is not in this list then it should cause an integration test to fail. +***********************************************************************************************************************************/ +#define COMPRESS_TYPE_REGEXP "(\\.gz|\\.lz4|\\.zst|\\.xz|\\.bz2)" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Get enum from a compression type string +CompressType compressTypeEnum(const String *type); + +// Check that a valid compress type is compiled into this binary. Errors when the compress type is not present. +void compressTypePresent(CompressType type); + +// Get string representation of a compression type. This is the the extension without the period. +const String *compressTypeStr(CompressType type); + +// Get compression type from a (file) name by checking the extension. If the extension is not a supported compression type then +// compressType none is returned, even if the file is compressed with some unknown type. +CompressType compressTypeFromName(const String *name); + +// Compression filter for the specified type. Error when compress type is none or invalid. +IoFilter *compressFilter(CompressType type, int level); + +// Compression/decompression filter based on string type and a parameter list. This is useful when a filter must be created on a +// remote system since the filter type and parameters can be passed through a protocol. +IoFilter *compressFilterVar(const String *filterType, const VariantList *filterParamList); + +// Decompression filter for the specified type. Error when compress type is none or invalid. +IoFilter *decompressFilter(CompressType type); + +// Get extension for the current compression type +const String *compressExtStr(CompressType type); + +// Add extension for current compression type to the file +void compressExtCat(String *file, CompressType type); + +// Remove the specified compression extension. Error when the extension is not correct. +String *compressExtStrip(const String *file, CompressType type); + +#endif diff --git a/src/common/compress/helper.intern.h b/src/common/compress/helper.intern.h new file mode 100644 index 000000000..900644f8b --- /dev/null +++ b/src/common/compress/helper.intern.h @@ -0,0 +1,15 @@ +/*********************************************************************************************************************************** +Compression Helper Internal +***********************************************************************************************************************************/ +#ifndef COMMON_COMPRESS_HELPER_INTERN_H +#define COMMON_COMPRESS_HELPER_INTERN_H + +#include "common/compress/helper.h" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Default compression level for a compression type, used while loading the configuration +int compressLevelDefault(CompressType type); + +#endif diff --git a/src/config/config.auto.c b/src/config/config.auto.c index 003454564..7face3b38 100644 --- a/src/config/config.auto.c +++ b/src/config/config.auto.c @@ -240,6 +240,7 @@ STRING_EXTERN(CFGOPT_CMD_SSH_STR, CFGOPT_CMD_S STRING_EXTERN(CFGOPT_COMPRESS_STR, CFGOPT_COMPRESS); STRING_EXTERN(CFGOPT_COMPRESS_LEVEL_STR, CFGOPT_COMPRESS_LEVEL); STRING_EXTERN(CFGOPT_COMPRESS_LEVEL_NETWORK_STR, CFGOPT_COMPRESS_LEVEL_NETWORK); +STRING_EXTERN(CFGOPT_COMPRESS_TYPE_STR, CFGOPT_COMPRESS_TYPE); STRING_EXTERN(CFGOPT_CONFIG_STR, CFGOPT_CONFIG); STRING_EXTERN(CFGOPT_CONFIG_INCLUDE_PATH_STR, CFGOPT_CONFIG_INCLUDE_PATH); STRING_EXTERN(CFGOPT_CONFIG_PATH_STR, CFGOPT_CONFIG_PATH); @@ -509,6 +510,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST CONFIG_OPTION_DEFINE_ID(cfgDefOptCompressLevelNetwork) ) + //------------------------------------------------------------------------------------------------------------------------------ + CONFIG_OPTION + ( + CONFIG_OPTION_NAME(CFGOPT_COMPRESS_TYPE) + CONFIG_OPTION_INDEX(0) + CONFIG_OPTION_DEFINE_ID(cfgDefOptCompressType) + ) + //------------------------------------------------------------------------------------------------------------------------------ CONFIG_OPTION ( diff --git a/src/config/config.auto.h b/src/config/config.auto.h index 0f5554bdf..201cd6147 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -71,6 +71,8 @@ Option constants STRING_DECLARE(CFGOPT_COMPRESS_LEVEL_STR); #define CFGOPT_COMPRESS_LEVEL_NETWORK "compress-level-network" STRING_DECLARE(CFGOPT_COMPRESS_LEVEL_NETWORK_STR); +#define CFGOPT_COMPRESS_TYPE "compress-type" + STRING_DECLARE(CFGOPT_COMPRESS_TYPE_STR); #define CFGOPT_CONFIG "config" STRING_DECLARE(CFGOPT_CONFIG_STR); #define CFGOPT_CONFIG_INCLUDE_PATH "config-include-path" @@ -390,7 +392,7 @@ Option constants #define CFGOPT_TYPE "type" STRING_DECLARE(CFGOPT_TYPE_STR); -#define CFG_OPTION_TOTAL 172 +#define CFG_OPTION_TOTAL 173 /*********************************************************************************************************************************** Command enum @@ -433,6 +435,7 @@ typedef enum cfgOptCompress, cfgOptCompressLevel, cfgOptCompressLevelNetwork, + cfgOptCompressType, cfgOptConfig, cfgOptConfigIncludePath, cfgOptConfigPath, diff --git a/src/config/define.auto.c b/src/config/define.auto.c index 662967712..e0285e55b 100644 --- a/src/config/define.auto.c +++ b/src/config/define.auto.c @@ -640,10 +640,12 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_SECURE(false) CFGDEFDATA_OPTION_HELP_SECTION("general") - CFGDEFDATA_OPTION_HELP_SUMMARY("Use gzip file compression.") + CFGDEFDATA_OPTION_HELP_SUMMARY("Use file compression.") CFGDEFDATA_OPTION_HELP_DESCRIPTION ( - "Backup files are compatible with command-line gzip tools." + "Backup files are compatible with command-line compression tools.\n" + "\n" + "This option is now deprecated. The compress-type option should be used instead." ) CFGDEFDATA_OPTION_COMMAND_LIST @@ -662,7 +664,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION ( CFGDEFDATA_OPTION_NAME("compress-level") - CFGDEFDATA_OPTION_REQUIRED(true) + CFGDEFDATA_OPTION_REQUIRED(false) CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal) CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeInteger) CFGDEFDATA_OPTION_INTERNAL(false) @@ -671,10 +673,10 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_SECURE(false) CFGDEFDATA_OPTION_HELP_SECTION("general") - CFGDEFDATA_OPTION_HELP_SUMMARY("Compression level for stored files.") + CFGDEFDATA_OPTION_HELP_SUMMARY("File compression level.") CFGDEFDATA_OPTION_HELP_DESCRIPTION ( - "Sets the zlib level to be used for file compression when compress=y." + "Sets the level to be used for file compression when compress-type does not equal none or compress=y (deprecated)." ) CFGDEFDATA_OPTION_COMMAND_LIST @@ -686,7 +688,6 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_OPTIONAL_LIST ( CFGDEFDATA_OPTION_OPTIONAL_ALLOW_RANGE(0, 9) - CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("6") ) ) @@ -703,13 +704,13 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST CFGDEFDATA_OPTION_SECURE(false) CFGDEFDATA_OPTION_HELP_SECTION("general") - CFGDEFDATA_OPTION_HELP_SUMMARY("Compression level for network transfer when compress=n.") + CFGDEFDATA_OPTION_HELP_SUMMARY("Network compression level.") CFGDEFDATA_OPTION_HELP_DESCRIPTION ( - "Sets the zlib level to be used for protocol compression when compress=n and the database cluster is not on the same " - "host as the repository. Protocol compression is used to reduce network traffic but can be disabled by setting " - "compress-level-network=0. When compress=y the compress-level-network setting is ignored and compress-level is " - "used instead so that the file is only compressed once. SSH compression is always disabled." + "Sets the network compression level when compress-type=none and the command is not run on the same host as the " + "repository. Compression is used to reduce network traffic but can be disabled by setting " + "compress-level-network=0. When compress-type does not equal none the compress-level-network setting is ignored " + "and compress-level is used instead so that the file is only compressed once. SSH compression is always disabled." ) CFGDEFDATA_OPTION_COMMAND_LIST @@ -733,6 +734,45 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST ) ) + // ----------------------------------------------------------------------------------------------------------------------------- + CFGDEFDATA_OPTION + ( + CFGDEFDATA_OPTION_NAME("compress-type") + CFGDEFDATA_OPTION_REQUIRED(true) + CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal) + CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeString) + CFGDEFDATA_OPTION_INTERNAL(false) + + CFGDEFDATA_OPTION_INDEX_TOTAL(1) + CFGDEFDATA_OPTION_SECURE(false) + + CFGDEFDATA_OPTION_HELP_SECTION("general") + CFGDEFDATA_OPTION_HELP_SUMMARY("File compression type.") + CFGDEFDATA_OPTION_HELP_DESCRIPTION + ( + "The following compression types are supported:\n" + "\n" + "* gz - gzip compression format" + ) + + CFGDEFDATA_OPTION_COMMAND_LIST + ( + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush) + CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup) + ) + + CFGDEFDATA_OPTION_OPTIONAL_LIST + ( + CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST + ( + "none", + "gz" + ) + + CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("gz") + ) + ) + // ----------------------------------------------------------------------------------------------------------------------------- CFGDEFDATA_OPTION ( diff --git a/src/config/define.auto.h b/src/config/define.auto.h index 20fcc0fe8..bdea46c46 100644 --- a/src/config/define.auto.h +++ b/src/config/define.auto.h @@ -61,6 +61,7 @@ typedef enum cfgDefOptCompress, cfgDefOptCompressLevel, cfgDefOptCompressLevelNetwork, + cfgDefOptCompressType, cfgDefOptConfig, cfgDefOptConfigIncludePath, cfgDefOptConfigPath, diff --git a/src/config/load.c b/src/config/load.c index 6d6e00bfc..d43397b8b 100644 --- a/src/config/load.c +++ b/src/config/load.c @@ -7,6 +7,7 @@ Configuration Load #include #include "command/command.h" +#include "common/compress/helper.intern.h" #include "common/memContext.h" #include "common/debug.h" #include "common/io/io.h" @@ -228,6 +229,41 @@ cfgLoadUpdateOption(void) strPtr(cfgOptionStr(cfgOptRepoS3Bucket))); } + // Check/update compress-type if compress is valid. There should be no references to the compress option outside this block. + if (cfgOptionValid(cfgOptCompress)) + { + if (cfgOptionSource(cfgOptCompress) != cfgSourceDefault) + { + if (cfgOptionSource(cfgOptCompressType) != cfgSourceDefault) + { + LOG_WARN( + "'" CFGOPT_COMPRESS "' and '" CFGOPT_COMPRESS_TYPE "' options should not both be set\n" + "HINT: '" CFGOPT_COMPRESS_TYPE "' is preferred and '" CFGOPT_COMPRESS "' is deprecated."); + } + + // Set compress-type to none. Eventually the compress option will be deprecated and removed so this reduces code churn + // when that happens. + if (!cfgOptionBool(cfgOptCompress) && cfgOptionSource(cfgOptCompressType) == cfgSourceDefault) + cfgOptionSet(cfgOptCompressType, cfgSourceParam, VARSTR(compressTypeStr(compressTypeNone))); + } + + // Now invalidate compress so it can't be used and won't be passed to child processes + cfgOptionValidSet(cfgOptCompress, false); + cfgOptionSet(cfgOptCompress, cfgSourceDefault, NULL); + } + + // Check that selected compress type has been compiled into this binary + if (cfgOptionValid(cfgOptCompressType)) + compressTypePresent(compressTypeEnum(cfgOptionStr(cfgOptCompressType))); + + // Update compress-level default based on the compression type + if (cfgOptionValid(cfgOptCompressLevel) && cfgOptionSource(cfgOptCompressLevel) == cfgSourceDefault) + { + cfgOptionSet( + cfgOptCompressLevel, cfgSourceDefault, + VARINT(compressLevelDefault(compressTypeEnum(cfgOptionStr(cfgOptCompressType))))); + } + FUNCTION_LOG_RETURN_VOID(); } diff --git a/src/config/parse.auto.c b/src/config/parse.auto.c index 59b70d122..82b554a18 100644 --- a/src/config/parse.auto.c +++ b/src/config/parse.auto.c @@ -192,6 +192,18 @@ static const struct option optionList[] = .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptCompressLevelNetwork, }, + // compress-type option + // ----------------------------------------------------------------------------------------------------------------------------- + { + .name = CFGOPT_COMPRESS_TYPE, + .has_arg = required_argument, + .val = PARSE_OPTION_FLAG | cfgOptCompressType, + }, + { + .name = "reset-" CFGOPT_COMPRESS_TYPE, + .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptCompressType, + }, + // config option // ----------------------------------------------------------------------------------------------------------------------------- { @@ -2338,6 +2350,7 @@ static const ConfigOption optionResolveOrder[] = cfgOptCompress, cfgOptCompressLevel, cfgOptCompressLevelNetwork, + cfgOptCompressType, cfgOptConfig, cfgOptConfigIncludePath, cfgOptConfigPath, diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index 5b268cd38..d129c6653 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -422,7 +422,7 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest) .optionBackupStandby = manData->backupOptionStandby != NULL ? varBool(manData->backupOptionStandby) : false, .optionChecksumPage = manData->backupOptionChecksumPage != NULL ? varBool(manData->backupOptionChecksumPage) : false, - .optionCompress = manData->backupOptionCompress, + .optionCompress = manData->backupOptionCompressType != compressTypeNone, .optionHardlink = manData->backupOptionHardLink, .optionOnline = manData->backupOptionOnline, }; diff --git a/src/info/manifest.c b/src/info/manifest.c index 37d8a7ca6..e95735772 100644 --- a/src/info/manifest.c +++ b/src/info/manifest.c @@ -129,6 +129,8 @@ STRING_STATIC(MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, "target:path STRING_STATIC(MANIFEST_KEY_OPTION_CHECKSUM_PAGE_STR, MANIFEST_KEY_OPTION_CHECKSUM_PAGE); #define MANIFEST_KEY_OPTION_COMPRESS "option-compress" STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_STR, MANIFEST_KEY_OPTION_COMPRESS); +#define MANIFEST_KEY_OPTION_COMPRESS_TYPE "option-compress-type" + STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_TYPE_STR, MANIFEST_KEY_OPTION_COMPRESS_TYPE); #define MANIFEST_KEY_OPTION_COMPRESS_LEVEL "option-compress-level" STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_LEVEL_STR, MANIFEST_KEY_OPTION_COMPRESS_LEVEL); #define MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK "option-compress-level-network" @@ -1030,13 +1032,13 @@ manifestNewBuild( /**********************************************************************************************************************************/ void -manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, bool compress) +manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, CompressType compressType) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(MANIFEST, this); FUNCTION_LOG_PARAM(BOOL, delta); FUNCTION_LOG_PARAM(TIME, copyStart); - FUNCTION_LOG_PARAM(TIME, compress); + FUNCTION_LOG_PARAM(ENUM, compressType); FUNCTION_LOG_END(); ASSERT(this != NULL); @@ -1054,7 +1056,7 @@ manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, bool compres // This value is not needed in this function, but it is needed for resumed manifests and this is last place to set it before // processing begins - this->data.backupOptionCompress = compress; + this->data.backupOptionCompressType = compressType; } MEM_CONTEXT_END(); @@ -1660,8 +1662,13 @@ manifestLoadCallback(void *callbackData, const String *section, const String *ke manifest->data.backupOptionArchiveCheck = jsonToBool(value); else if (strEq(key, MANIFEST_KEY_OPTION_ARCHIVE_COPY_STR)) manifest->data.backupOptionArchiveCopy = jsonToBool(value); + // Historically this option meant to add gz compression else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_STR)) - manifest->data.backupOptionCompress = jsonToBool(value); + manifest->data.backupOptionCompressType = jsonToBool(value) ? compressTypeGz : compressTypeNone; + // This new option allows any type of compression to be specified. It must be parsed after the option above so the + // value does not get overwritten. Since options are stored in alpha order this should always be true. + else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_TYPE_STR)) + manifest->data.backupOptionCompressType = compressTypeEnum(jsonToStr(value)); else if (strEq(key, MANIFEST_KEY_OPTION_HARDLINK_STR)) manifest->data.backupOptionHardLink = jsonToBool(value); else if (strEq(key, MANIFEST_KEY_OPTION_ONLINE_STR)) @@ -1944,9 +1951,11 @@ manifestSaveCallback(void *callbackData, const String *sectionNext, InfoSave *in jsonFromVar(manifest->data.backupOptionChecksumPage)); } + // Set the option when compression is turned on. In older versions this also implied gz compression but in newer versions + // the type option must also be set if compression is not gz. infoSaveValue( infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_COMPRESS_STR, - jsonFromBool(manifest->data.backupOptionCompress)); + jsonFromBool(manifest->data.backupOptionCompressType != compressTypeNone)); if (manifest->data.backupOptionCompressLevel != NULL) { @@ -1962,6 +1971,11 @@ manifestSaveCallback(void *callbackData, const String *sectionNext, InfoSave *in jsonFromVar(manifest->data.backupOptionCompressLevelNetwork)); } + // Set the compression type. Older versions will ignore this and assume gz compression if the compress option is set. + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_COMPRESS_TYPE_STR, + jsonFromStr(compressTypeStr(manifest->data.backupOptionCompressType))); + if (manifest->data.backupOptionDelta != NULL) { infoSaveValue( diff --git a/src/info/manifest.h b/src/info/manifest.h index 82a565379..9686b25f8 100644 --- a/src/info/manifest.h +++ b/src/info/manifest.h @@ -11,6 +11,7 @@ nothing is missing or corrupt. It is also useful for reporting, e.g. size of ba #define INFO_MANIFEST_H #include "command/backup/common.h" +#include "common/compress/helper.h" #include "common/crypto/common.h" #include "common/type/variantList.h" @@ -66,9 +67,9 @@ typedef struct ManifestData const Variant *backupOptionStandby; // Will the backup be performed from a standby? const Variant *backupOptionBufferSize; // Buffer size used for file/protocol operations const Variant *backupOptionChecksumPage; // Will page checksums be verified? - bool backupOptionCompress; // Will compression be used for backup? - const Variant *backupOptionCompressLevel; // Level to use for compression - const Variant *backupOptionCompressLevelNetwork; // Level to use for network compression + CompressType backupOptionCompressType; // Compression type used for the backup + const Variant *backupOptionCompressLevel; // Level used for compression (if type not none) + const Variant *backupOptionCompressLevelNetwork; // Level used for network compression const Variant *backupOptionDelta; // Will a checksum delta be performed? bool backupOptionHardLink; // Will hardlinks be created in the backup? bool backupOptionOnline; // Will an online backup be performed? @@ -161,7 +162,7 @@ Manifest *manifestNewLoad(IoRead *read); Build functions ***********************************************************************************************************************************/ // Validate the timestamps in the manifest given a copy start time, i.e. all times should be <= the copy start time -void manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, bool compress); +void manifestBuildValidate(Manifest *this, bool delta, time_t copyStart, CompressType compressType); // Create a diff/incr backup by comparing to a previous backup manifest void manifestBuildIncr(Manifest *this, const Manifest *prior, BackupType type, const String *archiveStart); diff --git a/src/storage/remote/protocol.c b/src/storage/remote/protocol.c index 178a12056..2641cb8d8 100644 --- a/src/storage/remote/protocol.c +++ b/src/storage/remote/protocol.c @@ -4,8 +4,7 @@ Remote Storage Protocol Handler #include "build.auto.h" #include "command/backup/pageChecksum.h" -#include "common/compress/gz/compress.h" -#include "common/compress/gz/decompress.h" +#include "common/compress/helper.h" #include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" @@ -72,10 +71,10 @@ storageRemoteFilterGroup(IoFilterGroup *filterGroup, const Variant *filterList) const String *filterKey = varStr(varLstGet(kvKeyList(filterKv), 0)); const VariantList *filterParam = varVarLst(kvGet(filterKv, VARSTR(filterKey))); - if (strEq(filterKey, GZ_COMPRESS_FILTER_TYPE_STR)) - ioFilterGroupAdd(filterGroup, gzCompressNewVar(filterParam)); - else if (strEq(filterKey, GZ_DECOMPRESS_FILTER_TYPE_STR)) - ioFilterGroupAdd(filterGroup, gzDecompressNew()); + IoFilter *filter = compressFilterVar(filterKey, filterParam); + + if (filter != NULL) + ioFilterGroupAdd(filterGroup, filter); else if (strEq(filterKey, CIPHER_BLOCK_FILTER_TYPE_STR)) ioFilterGroupAdd(filterGroup, cipherBlockNewVar(filterParam)); else if (strEq(filterKey, CRYPTO_HASH_FILTER_TYPE_STR)) diff --git a/src/storage/remote/read.c b/src/storage/remote/read.c index c4f4769a4..95e16785a 100644 --- a/src/storage/remote/read.c +++ b/src/storage/remote/read.c @@ -6,8 +6,7 @@ Remote Storage Read #include #include -#include "common/compress/gz/compress.h" -#include "common/compress/gz/decompress.h" +#include "common/compress/helper.h" #include "common/debug.h" #include "common/io/read.intern.h" #include "common/log.h" @@ -65,7 +64,10 @@ storageReadRemoteOpen(THIS_VOID) { // If the file is compressible add compression filter on the remote if (this->interface.compressible) - ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), gzCompressNew((int)this->interface.compressLevel)); + { + ioFilterGroupAdd( + ioReadFilterGroup(storageReadIo(this->read)), compressFilter(compressTypeGz, (int)this->interface.compressLevel)); + } ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR); protocolCommandParamAdd(command, VARSTR(this->interface.name)); @@ -79,7 +81,7 @@ storageReadRemoteOpen(THIS_VOID) // If the file is compressible add decompression filter locally if (this->interface.compressible) - ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), gzDecompressNew()); + ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilter(compressTypeGz)); } MEM_CONTEXT_TEMP_END(); diff --git a/src/storage/remote/write.c b/src/storage/remote/write.c index 6031daa64..0522b2e2a 100644 --- a/src/storage/remote/write.c +++ b/src/storage/remote/write.c @@ -3,8 +3,7 @@ Remote Storage File write ***********************************************************************************************************************************/ #include "build.auto.h" -#include "common/compress/gz/compress.h" -#include "common/compress/gz/decompress.h" +#include "common/compress/helper.h" #include "common/debug.h" #include "common/io/write.intern.h" #include "common/log.h" @@ -70,7 +69,7 @@ storageWriteRemoteOpen(THIS_VOID) { // If the file is compressible add decompression filter on the remote if (this->interface.compressible) - ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, gzDecompressNew()); + ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilter(compressTypeGz)); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE_STR); protocolCommandParamAdd(command, VARSTR(this->interface.name)); @@ -92,7 +91,11 @@ storageWriteRemoteOpen(THIS_VOID) // If the file is compressible add compression filter locally if (this->interface.compressible) - ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(this->write)), gzCompressNew((int)this->interface.compressLevel)); + { + ioFilterGroupAdd( + ioWriteFilterGroup(storageWriteIo(this->write)), + compressFilter(compressTypeGz, (int)this->interface.compressLevel)); + } // Set free callback to ensure remote file is freed memContextCallbackSet(this->memContext, storageWriteRemoteFreeResource, this); diff --git a/test/define.yaml b/test/define.yaml index 4c02da2fe..e8ff774c2 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -258,12 +258,13 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: compress - total: 3 + total: 2 coverage: common/compress/gz/common: full common/compress/gz/compress: full common/compress/gz/decompress: full + common/compress/helper: full # ---------------------------------------------------------------------------------------------------------------------------- - name: crypto diff --git a/test/expect/mock-all-001.log b/test/expect/mock-all-001.log index ec5cc8b7e..5765b9780 100644 --- a/test/expect/mock-all-001.log +++ b/test/expect/mock-all-001.log @@ -58,7 +58,7 @@ backrest-checksum="[CHECKSUM]" full backup - create pg_stat link, pg_clog dir (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --manifest-save-threshold=3 --buffer-size=[BUFFER-SIZE] --checksum-page --process-max=1 --repo1-type=cifs --type=full --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --checksum-page --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --manifest-save-threshold=3 --no-online --pg1-path=[TEST_PATH]/db-master/db/base --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --repo1-type=cifs --stanza=db --start-fast --type=full +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --checksum-page --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --manifest-save-threshold=3 --no-online --pg1-path=[TEST_PATH]/db-master/db/base --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --repo1-type=cifs --stanza=db --start-fast --type=full P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P01 INFO: backup file [TEST_PATH]/db-master/db/base/base/32768/33001 (64KB, 33%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b @@ -92,8 +92,8 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -139,6 +139,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=3 +option-compress-type="none" option-delta=false option-hardlink=false option-online=false @@ -239,7 +240,7 @@ P00 INFO: stop command end: completed successfully full backup - global stop (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --type=full --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 ERROR: [062]: stop file exists for all stanzas @@ -261,7 +262,7 @@ P00 INFO: stop command end: completed successfully full backup - stanza stop (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --type=full --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 ERROR: [062]: stop file exists for stanza db @@ -289,7 +290,7 @@ P00 INFO: start command end: completed successfully full backup - resume (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --force --checksum-page --delta --type=full --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --checksum-page --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --exclude=postgresql.auto.conf --exclude=pg_log/ --exclude=pg_log2 --exclude=apipe --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --checksum-page --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --exclude=postgresql.auto.conf --exclude=pg_log/ --exclude=pg_log2 --exclude=apipe --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 WARN: backup '[BACKUP-FULL-1]' missing manifest removed from backup.info @@ -334,8 +335,8 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -385,6 +386,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=3 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -754,7 +756,7 @@ restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf incr backup - add tablespace 1 (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] @@ -795,8 +797,8 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -843,6 +845,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=3 +option-compress-type="none" option-delta=false option-hardlink=false option-online=false @@ -945,7 +948,7 @@ backrest-checksum="[CHECKSUM]" incr backup - resume and add tablespace 2 (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --process-max=1 --delta --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 WARN: backup '[BACKUP-INCR-1]' missing manifest removed from backup.info @@ -1008,8 +1011,8 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -1056,6 +1059,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=3 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -1168,7 +1172,7 @@ backrest-checksum="[CHECKSUM]" diff backup - drop tablespace 11 (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --process-max=1 --delta --type=diff --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] @@ -1224,8 +1228,8 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -1272,6 +1276,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=3 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -1534,7 +1539,7 @@ restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf incr backup - add files and remove tablespace 2 (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --process-max=1 --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 INFO: last backup label = [BACKUP-DIFF-1], version = 0.00 @@ -1580,8 +1585,8 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -1628,6 +1633,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=3 +option-compress-type="none" option-delta=false option-hardlink=false option-online=false @@ -1733,7 +1739,7 @@ backrest-checksum="[CHECKSUM]" incr backup - update files (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 INFO: last backup label = [BACKUP-INCR-3], version = [VERSION-1] @@ -1798,8 +1804,8 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -1846,6 +1852,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=3 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -1952,7 +1959,7 @@ backrest-checksum="[CHECKSUM]" diff backup - updates since last full (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --process-max=1 --delta --type=diff --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] @@ -2013,8 +2020,8 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -2061,6 +2068,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=3 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -2168,11 +2176,11 @@ backrest-checksum="[CHECKSUM]" diff backup - remove files (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --process-max=1 --delta --type=diff --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --process-max=1 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --process-max=1 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] -P00 WARN: diff backup cannot alter compress option to 'true', reset to value in [BACKUP-FULL-2] +P00 WARN: diff backup cannot alter compress-type option to 'gz', reset to value in [BACKUP-FULL-2] P00 WARN: diff backup cannot alter hardlink option to 'true', reset to value in [BACKUP-FULL-2] P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 36%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b @@ -2228,7 +2236,6 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=y compress-level=3 db-timeout=45 lock-path=[TEST_PATH]/db-master/lock @@ -2277,6 +2284,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=3 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -2383,7 +2391,7 @@ backrest-checksum="[CHECKSUM]" full backup - update file (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --type=full --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 36%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b @@ -2425,7 +2433,6 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=y compress-level=3 db-timeout=45 lock-path=[TEST_PATH]/db-master/lock @@ -2473,6 +2480,7 @@ option-checksum-page=false option-compress=true option-compress-level=3 option-compress-level-network=3 +option-compress-type="gz" option-delta=false option-hardlink=true option-online=false @@ -2595,7 +2603,7 @@ P00 INFO: expire command end: completed successfully diff backup - add file (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --checksum-page --type=diff --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --checksum-page --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --checksum-page --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1] @@ -2639,7 +2647,6 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=y compress-level=3 db-timeout=45 lock-path=[TEST_PATH]/db-master/lock @@ -2688,6 +2695,7 @@ option-checksum-page=false option-compress=true option-compress-level=3 option-compress-level-network=3 +option-compress-type="gz" option-delta=false option-hardlink=true option-online=false @@ -3026,7 +3034,7 @@ restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf diff backup - option backup-standby reset - backup performed from master (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --log-level-console=info --backup-standby --type=diff --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --backup-standby --buffer-size=[BUFFER-SIZE] --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=info --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2/base --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff +P00 INFO: backup command begin [BACKREST-VERSION]: --backup-standby --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=info --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2/base --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 WARN: option backup-standby is enabled but backup is offline - backups will be performed from the primary @@ -3048,7 +3056,6 @@ pg1-path=[TEST_PATH]/db-master/db/base-2/base [global] buffer-size=[BUFFER-SIZE] -compress=y compress-level=3 db-timeout=45 lock-path=[TEST_PATH]/db-master/lock @@ -3097,6 +3104,7 @@ option-checksum-page=false option-compress=true option-compress-level=3 option-compress-level-network=3 +option-compress-type="gz" option-delta=false option-hardlink=true option-online=false diff --git a/test/expect/mock-all-002.log b/test/expect/mock-all-002.log index c4c4d3450..6cde89be4 100644 --- a/test/expect/mock-all-002.log +++ b/test/expect/mock-all-002.log @@ -62,9 +62,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -92,9 +92,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=warn @@ -149,6 +149,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=1 +option-compress-type="none" option-delta=false option-hardlink=false option-online=false @@ -254,9 +255,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -284,9 +285,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=warn @@ -345,6 +346,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=1 +option-compress-type="none" option-delta=false option-hardlink=false option-online=false @@ -485,9 +487,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -515,9 +517,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=warn @@ -573,6 +575,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=1 +option-compress-type="none" option-delta=false option-hardlink=false option-online=false @@ -689,9 +692,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -719,9 +722,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=warn @@ -777,6 +780,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=1 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -901,9 +905,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -931,9 +935,9 @@ pg1-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=warn @@ -989,6 +993,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=1 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -1130,9 +1135,9 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -1160,9 +1165,9 @@ pg1-path=[TEST_PATH]/db-master/db/base-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=warn @@ -1218,6 +1223,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=1 +option-compress-type="none" option-delta=false option-hardlink=false option-online=false @@ -1339,9 +1345,9 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -1369,9 +1375,9 @@ pg1-path=[TEST_PATH]/db-master/db/base-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=warn @@ -1427,6 +1433,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=1 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -1551,9 +1558,9 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -1581,9 +1588,9 @@ pg1-path=[TEST_PATH]/db-master/db/base-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=warn @@ -1639,6 +1646,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=1 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -1747,7 +1755,7 @@ diff backup - remove files (backup host) ------------------------------------------------------------------------------------------------------------------------------------ P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. -P00 WARN: diff backup cannot alter compress option to 'true', reset to value in [BACKUP-FULL-2] +P00 WARN: diff backup cannot alter compress-type option to 'gz', reset to value in [BACKUP-FULL-2] P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] P00 WARN: file 'changetime.txt' has timestamp earlier than prior backup, enabling delta checksum P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt: file size 12 is not divisible by page size 8192 @@ -1764,9 +1772,9 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -1794,7 +1802,6 @@ pg1-path=[TEST_PATH]/db-master/db/base-2 [global] buffer-size=[BUFFER-SIZE] -compress=y compress-level=3 compress-level-network=1 db-timeout=45 @@ -1852,6 +1859,7 @@ option-checksum-page=true option-compress=false option-compress-level=3 option-compress-level-network=1 +option-compress-type="none" option-delta=true option-hardlink=false option-online=false @@ -1971,9 +1979,9 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -2001,7 +2009,6 @@ pg1-path=[TEST_PATH]/db-master/db/base-2 [global] buffer-size=[BUFFER-SIZE] -compress=y compress-level=3 compress-level-network=1 db-timeout=45 @@ -2058,6 +2065,7 @@ option-checksum-page=false option-compress=true option-compress-level=3 option-compress-level-network=1 +option-compress-type="gz" option-delta=false option-hardlink=false option-online=false @@ -2183,9 +2191,9 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -2213,7 +2221,6 @@ pg1-path=[TEST_PATH]/db-master/db/base-2 [global] buffer-size=[BUFFER-SIZE] -compress=y compress-level=3 compress-level-network=1 db-timeout=45 @@ -2271,6 +2278,7 @@ option-checksum-page=false option-compress=true option-compress-level=3 option-compress-level-network=1 +option-compress-type="gz" option-delta=false option-hardlink=false option-online=false @@ -2410,7 +2418,7 @@ restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf diff backup - option backup-standby reset - backup performed from master (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=info --backup-standby --type=diff --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: backup command begin [BACKREST-VERSION]: --backup-standby --buffer-size=[BUFFER-SIZE] --compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=info --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base-2/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db --start-fast --type=diff +P00 INFO: backup command begin [BACKREST-VERSION]: --backup-standby --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=info --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base-2/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db --start-fast --type=diff P00 WARN: option repo1-retention-full is not set, the repository may run out of space HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum. P00 WARN: option backup-standby is enabled but backup is offline - backups will be performed from the primary @@ -2434,9 +2442,9 @@ pg1-path=[TEST_PATH]/db-master/db/base-2/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=warn @@ -2464,7 +2472,6 @@ pg1-path=[TEST_PATH]/db-master/db/base-2/base [global] buffer-size=[BUFFER-SIZE] -compress=y compress-level=3 compress-level-network=1 db-timeout=45 @@ -2522,6 +2529,7 @@ option-checksum-page=false option-compress=true option-compress-level=3 option-compress-level-network=1 +option-compress-type="gz" option-delta=false option-hardlink=false option-online=false diff --git a/test/expect/mock-archive-001.log b/test/expect/mock-archive-001.log index d2278115a..71d23ac5e 100644 --- a/test/expect/mock-archive-001.log +++ b/test/expect/mock-archive-001.log @@ -1,9 +1,9 @@ -run 001 - rmt 0, s3 0, enc 1 -============================ +run 001 - rmt 0, s3 0, enc 1, cmp gz +==================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [055]: unable to load info file '[TEST_PATH]/db-master/repo/archive/db/archive.info' or '[TEST_PATH]/db-master/repo/archive/db/archive.info.copy': FileMissingError: unable to open missing file '[TEST_PATH]/db-master/repo/archive/db/archive.info' for read FileMissingError: unable to open missing file '[TEST_PATH]/db-master/repo/archive/db/archive.info.copy' for read @@ -73,9 +73,9 @@ db-version="9.4" [backrest] backrest-checksum="[CHECKSUM]" -> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress-type=gz [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=gz --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: pushed WAL file '000000010000000100000001' to the archive P00 INFO: archive-push command end: completed successfully @@ -91,15 +91,15 @@ P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000 P00 INFO: found 000000010000000100000001 in the archive P00 INFO: archive-get command end: completed successfully -> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress-type=gz --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --buffer-size=[BUFFER-SIZE] --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=gz --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db P00 INFO: pushed WAL file '000000010000000100000002' to the archive asynchronously P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --archive-async [TEST_PATH]/db-master/db/base/pg_xlog/00000002.history ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/00000002.history] --archive-async --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/00000002.history] --archive-async --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db P00 INFO: pushed WAL file '00000002.history' to the archive asynchronously P00 INFO: archive-push command end: completed successfully @@ -111,7 +111,7 @@ P00 INFO: archive-get command end: aborted with exception [044] > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [044]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 9.4, system-id 5000900090001855000 HINT: are you archiving to the correct stanza? P00 INFO: archive-push command end: aborted with exception [044] @@ -130,7 +130,7 @@ P00 INFO: stop command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [062]: stop file exists for stanza db P00 INFO: archive-push command end: aborted with exception [062] @@ -148,7 +148,7 @@ P00 INFO: start command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 WARN: WAL file '000000010000000100000002' already exists in the archive with the same checksum HINT: this is valid in some recovery scenarios but may also indicate a problem. P00 INFO: pushed WAL file '000000010000000100000002' to the archive @@ -156,7 +156,7 @@ P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [045]: WAL file '000000010000000100000002' already exists in the archive P00 INFO: archive-push command end: aborted with exception [045] @@ -180,13 +180,13 @@ P00 INFO: archive-get command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: pushed WAL file '000000010000000100000002.partial' to the archive P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 WARN: WAL file '000000010000000100000002.partial' already exists in the archive with the same checksum HINT: this is valid in some recovery scenarios but may also indicate a problem. P00 INFO: pushed WAL file '000000010000000100000002.partial' to the archive @@ -194,6 +194,6 @@ P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [045]: WAL file '000000010000000100000002.partial' already exists in the archive P00 INFO: archive-push command end: aborted with exception [045] diff --git a/test/expect/mock-archive-002.log b/test/expect/mock-archive-002.log index aef285f99..247c3daf2 100644 --- a/test/expect/mock-archive-002.log +++ b/test/expect/mock-archive-002.log @@ -1,9 +1,9 @@ -run 002 - rmt 1, s3 1, enc 0 -============================ +run 002 - rmt 1, s3 1, enc 0, cmp gz +==================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 ERROR: [055]: unable to load info file '/archive/db/archive.info' or '/archive/db/archive.info.copy': FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info': No such file or directory FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info.copy': No such file or directory @@ -69,9 +69,9 @@ db-version="9.4" [backrest] backrest-checksum="[CHECKSUM]" -> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --cmd-ssh=/usr/bin/ssh --compress [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --cmd-ssh=/usr/bin/ssh --compress-type=gz [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --cmd-ssh=/usr/bin/ssh --compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --cmd-ssh=/usr/bin/ssh --compress-level=3 --compress-level-network=1 --compress-type=gz --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 INFO: pushed WAL file '000000010000000100000001' to the archive P00 INFO: archive-push command end: completed successfully @@ -87,15 +87,15 @@ P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000 P00 INFO: found 000000010000000100000001 in the archive P00 INFO: archive-get command end: completed successfully -> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress-type=gz --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --buffer-size=[BUFFER-SIZE] --compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=gz --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --spool-path=[TEST_PATH]/db-master/spool --stanza=db P00 INFO: pushed WAL file '000000010000000100000002' to the archive asynchronously P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --archive-async [TEST_PATH]/db-master/db/base/pg_xlog/00000002.history ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/00000002.history] --archive-async --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/00000002.history] --archive-async --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --spool-path=[TEST_PATH]/db-master/spool --stanza=db P00 INFO: pushed WAL file '00000002.history' to the archive asynchronously P00 INFO: archive-push command end: completed successfully @@ -107,7 +107,7 @@ P00 INFO: archive-get command end: aborted with exception [044] > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 ERROR: [044]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 9.4, system-id 5000900090001855000 HINT: are you archiving to the correct stanza? P00 INFO: archive-push command end: aborted with exception [044] @@ -126,7 +126,7 @@ P00 INFO: stop command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 ERROR: [062]: stop file exists for stanza db P00 INFO: archive-push command end: aborted with exception [062] @@ -144,7 +144,7 @@ P00 INFO: start command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 WARN: WAL file '000000010000000100000002' already exists in the archive with the same checksum HINT: this is valid in some recovery scenarios but may also indicate a problem. P00 INFO: pushed WAL file '000000010000000100000002' to the archive @@ -152,7 +152,7 @@ P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 ERROR: [045]: WAL file '000000010000000100000002' already exists in the archive P00 INFO: archive-push command end: aborted with exception [045] @@ -176,13 +176,13 @@ P00 INFO: archive-get command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 INFO: pushed WAL file '000000010000000100000002.partial' to the archive P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 WARN: WAL file '000000010000000100000002.partial' already exists in the archive with the same checksum HINT: this is valid in some recovery scenarios but may also indicate a problem. P00 INFO: pushed WAL file '000000010000000100000002.partial' to the archive @@ -190,6 +190,6 @@ P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=none --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 ERROR: [045]: WAL file '000000010000000100000002.partial' already exists in the archive P00 INFO: archive-push command end: aborted with exception [045] diff --git a/test/expect/mock-archive-stop-001.log b/test/expect/mock-archive-stop-001.log index c54b05edd..ce14afb1c 100644 --- a/test/expect/mock-archive-stop-001.log +++ b/test/expect/mock-archive-stop-001.log @@ -1,5 +1,5 @@ -run 001 - rmt 0, cmp 0, error version, s3 0, enc 1 -================================================== +run 001 - rmt 0, cmp none, error version, s3 0, enc 1 +===================================================== stanza-create db - create required data for stanza (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --no-online stanza-create diff --git a/test/expect/mock-archive-stop-002.log b/test/expect/mock-archive-stop-002.log index 2e1287552..450c143c6 100644 --- a/test/expect/mock-archive-stop-002.log +++ b/test/expect/mock-archive-stop-002.log @@ -1,5 +1,5 @@ -run 002 - rmt 1, cmp 1, error connect, s3 1, enc 0 -================================================== +run 002 - rmt 1, cmp gz, error connect, s3 1, enc 0 +=================================================== stanza-create db - create required data for stanza (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-create diff --git a/test/expect/mock-stanza-001.log b/test/expect/mock-stanza-001.log index 19e5be65a..971d80183 100644 --- a/test/expect/mock-stanza-001.log +++ b/test/expect/mock-stanza-001.log @@ -1,5 +1,5 @@ -run 001 - remote 0, s3 0, enc 0 -=============================== +run 001 - remote 0, s3 0, enc 0, cmp gz +======================================= stanza-create db - fail on missing control file (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --no-online --log-level-file=[LOG-LEVEL-FILE] stanza-create diff --git a/test/expect/mock-stanza-002.log b/test/expect/mock-stanza-002.log index d93c4986a..9e7591a96 100644 --- a/test/expect/mock-stanza-002.log +++ b/test/expect/mock-stanza-002.log @@ -1,5 +1,5 @@ -run 002 - remote 1, s3 1, enc 1 -=============================== +run 002 - remote 1, s3 1, enc 1, cmp gz +======================================= stanza-create db - fail on missing control file (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online --log-level-file=[LOG-LEVEL-FILE] stanza-create diff --git a/test/expect/real-all-001.log b/test/expect/real-all-001.log index 6ca78b82d..395952b32 100644 --- a/test/expect/real-all-001.log +++ b/test/expect/real-all-001.log @@ -1,5 +1,5 @@ -run 001 - bkp 0, sby 0, dst db-master, cmp 0, s3 0, enc 0 -========================================================= +run 001 - bkp 0, sby 0, dst db-master, cmp none, s3 0, enc 0 +============================================================ stanza-create db - main create stanza info files (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db stanza-create @@ -82,8 +82,8 @@ pg1-socket-path=[TEST_PATH]/db-master/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -130,8 +130,8 @@ pg1-socket-path=[TEST_PATH]/db-master/db [global] archive-async=y buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -265,8 +265,8 @@ pg1-socket-path=[TEST_PATH]/db-master/db [global] archive-async=y buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail diff --git a/test/expect/real-all-002.log b/test/expect/real-all-002.log index 768d4e2cd..b43a5d836 100644 --- a/test/expect/real-all-002.log +++ b/test/expect/real-all-002.log @@ -1,5 +1,5 @@ -run 002 - bkp 0, sby 1, dst db-master, cmp 0, s3 0, enc 0 -========================================================= +run 002 - bkp 0, sby 1, dst db-master, cmp none, s3 0, enc 0 +============================================================ stanza-create db - main create stanza info files (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db stanza-create @@ -28,8 +28,8 @@ pg8-path=[TEST_PATH]/db-standby/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -55,8 +55,8 @@ pg1-socket-path=[TEST_PATH]/db-standby/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-standby/lock log-level-console=detail @@ -111,8 +111,8 @@ pg8-path=[TEST_PATH]/db-standby/db/base [global] archive-async=y buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -140,8 +140,8 @@ pg1-socket-path=[TEST_PATH]/db-standby/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-standby/lock log-level-console=detail @@ -185,8 +185,8 @@ pg8-path=[TEST_PATH]/db-standby/db/base [global] archive-async=y buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -214,8 +214,8 @@ pg1-socket-path=[TEST_PATH]/db-standby/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-standby/lock log-level-console=detail diff --git a/test/expect/real-all-003.log b/test/expect/real-all-003.log index c914b8d9f..e0042c668 100644 --- a/test/expect/real-all-003.log +++ b/test/expect/real-all-003.log @@ -1,5 +1,5 @@ -run 003 - bkp 0, sby 1, dst db-standby, cmp 0, s3 0, enc 0 -========================================================== +run 003 - bkp 0, sby 1, dst db-standby, cmp none, s3 0, enc 0 +============================================================= stanza-create db - main create stanza info files (db-standby host) > [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --stanza=db stanza-create @@ -18,9 +18,9 @@ pg1-socket-path=[TEST_PATH]/db-master/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -55,9 +55,9 @@ pg8-path=[TEST_PATH]/db-master/db/base [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-standby/lock log-level-console=detail @@ -102,9 +102,9 @@ pg1-socket-path=[TEST_PATH]/db-master/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -142,9 +142,9 @@ pg8-path=[TEST_PATH]/db-master/db/base [global] archive-async=y buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-standby/lock log-level-console=detail @@ -174,9 +174,9 @@ pg1-socket-path=[TEST_PATH]/db-master/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -214,9 +214,9 @@ pg8-path=[TEST_PATH]/db-master/db/base [global] archive-async=y buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-standby/lock log-level-console=detail diff --git a/test/expect/real-all-004.log b/test/expect/real-all-004.log index 2050bfb66..c873a9727 100644 --- a/test/expect/real-all-004.log +++ b/test/expect/real-all-004.log @@ -1,5 +1,5 @@ -run 004 - bkp 1, sby 0, dst backup, cmp 1, s3 0, enc 1 -====================================================== +run 004 - bkp 1, sby 0, dst backup, cmp gz, s3 0, enc 1 +======================================================= stanza-create db - main create stanza info files (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stanza-create diff --git a/test/expect/real-all-005.log b/test/expect/real-all-005.log index 072c9b86e..bd1710e5c 100644 --- a/test/expect/real-all-005.log +++ b/test/expect/real-all-005.log @@ -1,5 +1,5 @@ -run 005 - bkp 1, sby 1, dst backup, cmp , s3 0, enc 0 -===================================================== +run 005 - bkp 1, sby 1, dst backup, cmp none, s3 0, enc 0 +========================================================= stanza-create db - main create stanza info files (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stanza-create @@ -18,9 +18,9 @@ pg1-socket-path=[TEST_PATH]/db-master/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -45,9 +45,9 @@ pg1-socket-path=[TEST_PATH]/db-standby/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-standby/lock log-level-console=detail @@ -86,9 +86,9 @@ pg8-port=6544 [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=detail @@ -132,9 +132,9 @@ pg1-socket-path=[TEST_PATH]/db-master/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -161,9 +161,9 @@ pg1-socket-path=[TEST_PATH]/db-standby/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-standby/lock log-level-console=detail @@ -203,9 +203,9 @@ pg8-port=6544 [global] archive-async=y buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=detail @@ -238,9 +238,9 @@ pg1-socket-path=[TEST_PATH]/db-master/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-master/lock log-level-console=detail @@ -267,9 +267,9 @@ pg1-socket-path=[TEST_PATH]/db-standby/db [global] buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/db-standby/lock log-level-console=detail @@ -309,9 +309,9 @@ pg8-port=6544 [global] archive-async=y buffer-size=[BUFFER-SIZE] -compress=n compress-level=3 compress-level-network=1 +compress-type=none db-timeout=45 lock-path=[TEST_PATH]/backup/lock log-level-console=detail diff --git a/test/expect/real-all-006.log b/test/expect/real-all-006.log index ecdb630f1..70ee17fe1 100644 --- a/test/expect/real-all-006.log +++ b/test/expect/real-all-006.log @@ -1,5 +1,5 @@ -run 006 - bkp 1, sby 0, dst backup, cmp 1, s3 1, enc 0 -====================================================== +run 006 - bkp 1, sby 0, dst backup, cmp gz, s3 1, enc 0 +======================================================= stanza-create db - main create stanza info files (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stanza-create diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm index ccdab71ff..df78f91b5 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm @@ -476,9 +476,9 @@ sub backupCompare foreach my $strFileKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE)) { # Determine repo size if compression or encryption is enabled - my $bCompressed = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS}; + my $strCompressType = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}; - if ($bCompressed || + if ($strCompressType ne CFGOPTVAL_COMPRESS_TYPE_NONE || (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}))) { @@ -487,7 +487,8 @@ sub backupCompare $oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ? $oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) : (storageRepo()->info(STORAGE_REPO_BACKUP . - "/${strBackup}/${strFileKey}" . ($bCompressed ? '.gz' : '')))->{size}; + "/${strBackup}/${strFileKey}" . + ($strCompressType eq CFGOPTVAL_COMPRESS_TYPE_NONE ? '' : ".${strCompressType}")))->{size}; if (defined($lRepoSize) && $lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE}) @@ -1031,9 +1032,9 @@ sub configCreate $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_LEVEL_NETWORK)} = 1; } - if (defined($$oParam{bCompress}) && !$$oParam{bCompress}) + if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne CFGOPTVAL_COMPRESS_TYPE_GZ) { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS)} = 'n'; + $oParamHash{&CFGDEF_SECTION_GLOBAL}{cfgOptionName(CFGOPT_COMPRESS_TYPE)} = $oParam->{strCompressType}; } if ($self->isHostBackup()) @@ -1200,7 +1201,14 @@ sub configUpdate { foreach my $strKey (keys(%{$hParam->{$strSection}})) { - $oConfig->{$strSection}{$strKey} = $hParam->{$strSection}{$strKey}; + if (defined($hParam->{$strSection}{$strKey})) + { + $oConfig->{$strSection}{$strKey} = $hParam->{$strSection}{$strKey}; + } + else + { + delete($oConfig->{$strSection}{$strKey}); + } } } @@ -1792,6 +1800,8 @@ sub restoreCompare $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}, $oTablespaceMap); $oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef, $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}); + $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_TYPE, undef, + $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}); my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); diff --git a/test/lib/pgBackRestTest/Env/HostEnvTest.pm b/test/lib/pgBackRestTest/Env/HostEnvTest.pm index de3fe3fe9..e13a287ee 100644 --- a/test/lib/pgBackRestTest/Env/HostEnvTest.pm +++ b/test/lib/pgBackRestTest/Env/HostEnvTest.pm @@ -43,6 +43,11 @@ use constant ENCRYPTION_KEY_MANIFEST => 'manifest'; use constant ENCRYPTION_KEY_BACKUPSET => 'backupset'; push @EXPORT, qw(ENCRYPTION_KEY_BACKUPSET); +use constant NONE => CFGOPTVAL_COMPRESS_TYPE_NONE; + push @EXPORT, qw(NONE); +use constant GZ => CFGOPTVAL_COMPRESS_TYPE_GZ; + push @EXPORT, qw(GZ); + #################################################################################################################################### # setup #################################################################################################################################### @@ -127,7 +132,7 @@ sub setup # Create db master config $oHostDbMaster->configCreate({ strBackupSource => $$oConfigParam{strBackupSource}, - bCompress => $$oConfigParam{bCompress}, + strCompressType => $$oConfigParam{strCompressType}, bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink}, bArchiveAsync => $$oConfigParam{bArchiveAsync}, bS3 => $$oConfigParam{bS3}}); @@ -136,7 +141,7 @@ sub setup if (defined($oHostBackup)) { $oHostBackup->configCreate({ - bCompress => $$oConfigParam{bCompress}, + strCompressType => $$oConfigParam{strCompressType}, bHardlink => $$oConfigParam{bHardLink}, bS3 => $$oConfigParam{bS3}}); } @@ -151,7 +156,7 @@ sub setup { $oHostDbStandby->configCreate({ strBackupSource => $$oConfigParam{strBackupSource}, - bCompress => $$oConfigParam{bCompress}, + strCompressType => $$oConfigParam{strCompressType}, bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink}, bArchiveAsync => $$oConfigParam{bArchiveAsync}}); } diff --git a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm index 6a9cfbd96..41010c4c3 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm @@ -62,14 +62,14 @@ sub run foreach my $rhRun ( - {vm => VM1, remote => false, s3 => true, encrypt => false, delta => true}, - {vm => VM1, remote => true, s3 => false, encrypt => true, delta => false}, - {vm => VM2, remote => false, s3 => false, encrypt => true, delta => true}, - {vm => VM2, remote => true, s3 => true, encrypt => false, delta => false}, - {vm => VM3, remote => false, s3 => false, encrypt => false, delta => true}, - {vm => VM3, remote => true, s3 => true, encrypt => true, delta => false}, - {vm => VM4, remote => false, s3 => false, encrypt => false, delta => false}, - {vm => VM4, remote => true, s3 => true, encrypt => true, delta => true}, + {vm => VM1, remote => false, s3 => true, encrypt => false, delta => true, compress => GZ}, + {vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => GZ}, + {vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => GZ}, + {vm => VM2, remote => true, s3 => true, encrypt => false, delta => false, compress => GZ}, + {vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => GZ}, + {vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => GZ}, + {vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => GZ}, + {vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => GZ}, ) { # Only run tests for this vm @@ -80,13 +80,14 @@ sub run my $bS3 = $rhRun->{s3}; my $bEncrypt = $rhRun->{encrypt}; my $bDeltaBackup = $rhRun->{delta}; + my $strCompressType = $rhRun->{compress}; # Increment the run, log, and decide whether this unit test should be run if (!$self->begin("rmt ${bRemote}, s3 ${bS3}, enc ${bEncrypt}, delta ${bDeltaBackup}")) {next} # Create hosts, file object, and config my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote, bCompress => false, bS3 => $bS3, bRepoEncrypt => $bEncrypt}); + true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => NONE}); # If S3 set process max to 2. This seems like the best place for parallel testing since it will help speed S3 processing # without slowing down the other tests too much. @@ -114,6 +115,7 @@ sub run $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 16384; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE} = JSON::PP::true; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::false; + $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = CFGOPTVAL_COMPRESS_TYPE_NONE; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL} = 3; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK} = $bRemote ? 1 : 3; $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = JSON::PP::false; @@ -809,8 +811,15 @@ sub run $strType = CFGOPTVAL_BACKUP_TYPE_DIFF; - # Enable compression to ensure a warning is raised - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS) => 'y'}}); + # Enable compression to ensure a warning is raised (reset when gz to avoid log churn since it is the default) + if ($strCompressType eq GZ) + { + $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => undef}}); + } + else + { + $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_COMPRESS_TYPE) => $strCompressType}}); + } # Enable hardlinks (except for s3) to ensure a warning is raised if (!$bS3) @@ -841,6 +850,7 @@ sub run # Now the compression and hardlink changes will take effect $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::true; + $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = $strCompressType; if (!$bS3) { diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm index a129d155c..9df346a0f 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm @@ -44,14 +44,14 @@ sub run foreach my $rhRun ( - {vm => VM1, remote => false, s3 => false, encrypt => false, compress => true, error => 0}, - {vm => VM1, remote => true, s3 => true, encrypt => true, compress => false, error => 1}, - {vm => VM2, remote => false, s3 => true, encrypt => false, compress => false, error => 0}, - {vm => VM2, remote => true, s3 => false, encrypt => true, compress => true, error => 0}, - {vm => VM3, remote => false, s3 => false, encrypt => true, compress => false, error => 0}, - {vm => VM3, remote => true, s3 => true, encrypt => false, compress => true, error => 1}, - {vm => VM4, remote => false, s3 => true, encrypt => true, compress => true, error => 0}, - {vm => VM4, remote => true, s3 => false, encrypt => false, compress => false, error => 0}, + {vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ, error => 0}, + {vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ, error => 1}, + {vm => VM2, remote => false, s3 => true, encrypt => false, compress => NONE, error => 0}, + {vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ, error => 0}, + {vm => VM3, remote => false, s3 => false, encrypt => true, compress => NONE, error => 0}, + {vm => VM3, remote => true, s3 => true, encrypt => false, compress => GZ, error => 1}, + {vm => VM4, remote => false, s3 => true, encrypt => true, compress => GZ, error => 0}, + {vm => VM4, remote => true, s3 => false, encrypt => false, compress => NONE, error => 0}, ) { # Only run tests for this vm @@ -61,20 +61,21 @@ sub run my $bRemote = $rhRun->{remote}; my $bS3 = $rhRun->{s3}; my $bEncrypt = $rhRun->{encrypt}; - my $bCompress = $rhRun->{compress}; + my $strCompressType = $rhRun->{compress}; my $iError = $rhRun->{error}; # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, error " . ($iError ? 'connect' : 'version') . ", s3 ${bS3}, " . - "enc ${bEncrypt}")) {next} + if (!$self->begin( + "rmt ${bRemote}, cmp ${strCompressType}, error " . ($iError ? 'connect' : 'version') . + ", s3 ${bS3}, enc ${bEncrypt}")) {next} # Create hosts, file object, and config my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true, bS3 => $bS3, - bRepoEncrypt => $bEncrypt}); + true, $self->expect(), {bHostBackup => $bRemote, strCompressType => $strCompressType, bArchiveAsync => true, + bS3 => $bS3, bRepoEncrypt => $bEncrypt}); # Create compression extension - my $strCompressExt = $bCompress ? qw{.} . COMPRESS_EXT : ''; + my $strCompressExt = $strCompressType ne NONE ? ".${strCompressType}" : ''; # Create the wal path my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog'; diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm index 9ad3f5b35..3c6d880ff 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm @@ -40,15 +40,15 @@ sub archiveCheck my $self = shift; my $strArchiveFile = shift; my $strArchiveChecksum = shift; - my $bCompress = shift; + my $strCompressType = shift; my $strSpoolPath = shift; # Build the archive name to check for at the destination my $strArchiveCheck = PG_VERSION_94 . "-1/${strArchiveFile}-${strArchiveChecksum}"; - if ($bCompress) + if (defined($strCompressType)) { - $strArchiveCheck .= '.gz'; + $strArchiveCheck .= ".${strCompressType}"; } my $oWait = waitInit(5); @@ -82,14 +82,14 @@ sub run foreach my $rhRun ( - {vm => VM1, remote => false, s3 => false, encrypt => false}, - {vm => VM1, remote => true, s3 => true, encrypt => true}, - {vm => VM2, remote => false, s3 => true, encrypt => false}, - {vm => VM2, remote => true, s3 => false, encrypt => true}, - {vm => VM3, remote => false, s3 => false, encrypt => true}, - {vm => VM3, remote => true, s3 => true, encrypt => false}, - {vm => VM4, remote => false, s3 => true, encrypt => true}, - {vm => VM4, remote => true, s3 => false, encrypt => false}, + {vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ}, + {vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ}, + {vm => VM2, remote => false, s3 => true, encrypt => false, compress => GZ}, + {vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ}, + {vm => VM3, remote => false, s3 => false, encrypt => true, compress => GZ}, + {vm => VM3, remote => true, s3 => true, encrypt => false, compress => GZ}, + {vm => VM4, remote => false, s3 => true, encrypt => true, compress => GZ}, + {vm => VM4, remote => true, s3 => false, encrypt => false, compress => GZ}, ) { # Only run tests for this vm @@ -99,12 +99,13 @@ sub run my $bRemote = $rhRun->{remote}; my $bS3 = $rhRun->{s3}; my $bEncrypt = $rhRun->{encrypt}; + my $strCompressType = $rhRun->{compress}; - if (!$self->begin("rmt ${bRemote}, s3 ${bS3}, enc ${bEncrypt}")) {next} + if (!$self->begin("rmt ${bRemote}, s3 ${bS3}, enc ${bEncrypt}, cmp ${strCompressType}")) {next} # Create hosts, file object, and config my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote, bCompress => false, bS3 => $bS3, bRepoEncrypt => $bEncrypt}); + true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, strCompressType => NONE}); # Reduce console logging to detail $oHostDbMaster->configUpdate({&CFGDEF_SECTION_GLOBAL => {cfgOptionName(CFGOPT_LOG_LEVEL_CONSOLE) => lc(DETAIL)}}); @@ -153,12 +154,13 @@ sub run my $strArchiveFile = $self->walGenerate($strWalPath, PG_VERSION_94, 2, $strSourceFile); $oHostDbMaster->executeSimple( - $strCommandPush . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . " --compress ${strWalPath}/${strSourceFile}", + $strCommandPush . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . + " --compress-type=${strCompressType} ${strWalPath}/${strSourceFile}", {oLogTest => $self->expect()}); - push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.gz"); + push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.${strCompressType}"); # Test that the WAL was pushed - $self->archiveCheck($strSourceFile, $strArchiveChecksum, true); + $self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType); # Remove from archive_status storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready"); @@ -211,7 +213,7 @@ sub run $strArchiveTmp = $oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' . - substr($strSourceFile, 0, 16) . "/${strSourceFile}-${strArchiveChecksum}." . COMPRESS_EXT . qw{.} . + substr($strSourceFile, 0, 16) . "/${strSourceFile}-${strArchiveChecksum}.${strCompressType}" . qw{.} . STORAGE_TEMP_EXT; storageTest()->put($strArchiveTmp, 'JUNK'); @@ -219,9 +221,10 @@ sub run # Push the WAL $oHostDbMaster->executeSimple( - "${strCommandPush} --compress --archive-async --process-max=2 ${strWalPath}/${strSourceFile}", + "${strCommandPush} --compress-type=${strCompressType} --archive-async --process-max=2" . + " ${strWalPath}/${strSourceFile}", {oLogTest => $self->expect()}); - push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}." . COMPRESS_EXT); + push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.${strCompressType}"); # Make sure the temp file no longer exists if it was created if (defined($strArchiveTmp)) @@ -242,7 +245,7 @@ sub run } # Test that the WAL was pushed - $self->archiveCheck($strSourceFile, $strArchiveChecksum, true, $oHostDbMaster->spoolPath()); + $self->archiveCheck($strSourceFile, $strArchiveChecksum, $strCompressType, $oHostDbMaster->spoolPath()); # Remove from archive_status storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready"); @@ -381,7 +384,7 @@ sub run $oHostDbMaster->executeSimple( $strCommandPush . " ${strWalPath}/${strSourceFile}.partial", {oLogTest => $self->expect()}); - $self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum, false); + $self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum); push(@stryExpectedWAL, "${strSourceFile}.partial-${strArchiveChecksum}"); @@ -390,8 +393,7 @@ sub run $oHostDbMaster->executeSimple( $strCommandPush . " ${strWalPath}/${strSourceFile}.partial", {oLogTest => $self->expect()}); - $self->archiveCheck( - "${strSourceFile}.partial", $strArchiveChecksum, false); + $self->archiveCheck("${strSourceFile}.partial", $strArchiveChecksum); #--------------------------------------------------------------------------------------------------------------------------- &log(INFO, ' .partial WAL with different checksum'); diff --git a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm index 113b3f4a0..8405a7d70 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm @@ -53,14 +53,14 @@ sub run foreach my $rhRun ( - {vm => VM1, remote => false, s3 => false, encrypt => true}, - {vm => VM1, remote => true, s3 => true, encrypt => false}, - {vm => VM2, remote => false, s3 => true, encrypt => true}, - {vm => VM2, remote => true, s3 => false, encrypt => false}, - {vm => VM3, remote => false, s3 => false, encrypt => false}, - {vm => VM3, remote => true, s3 => true, encrypt => true}, - {vm => VM4, remote => false, s3 => true, encrypt => false}, - {vm => VM4, remote => true, s3 => false, encrypt => true}, + {vm => VM1, remote => false, s3 => false, encrypt => true, compress => GZ}, + {vm => VM1, remote => true, s3 => true, encrypt => false, compress => GZ}, + {vm => VM2, remote => false, s3 => true, encrypt => true, compress => GZ}, + {vm => VM2, remote => true, s3 => false, encrypt => false, compress => GZ}, + {vm => VM3, remote => false, s3 => false, encrypt => false, compress => GZ}, + {vm => VM3, remote => true, s3 => true, encrypt => true, compress => GZ}, + {vm => VM4, remote => false, s3 => true, encrypt => false, compress => GZ}, + {vm => VM4, remote => true, s3 => false, encrypt => true, compress => GZ}, ) { # Only run tests for this vm @@ -70,13 +70,15 @@ sub run my $bRemote = $rhRun->{remote}; my $bS3 = $rhRun->{s3}; my $bEncrypt = $rhRun->{encrypt}; + my $strCompressType = $rhRun->{compress}; # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin("remote ${bRemote}, s3 ${bS3}, enc ${bEncrypt}")) {next} + if (!$self->begin("remote ${bRemote}, s3 ${bS3}, enc ${bEncrypt}, cmp ${strCompressType}")) {next} # Create hosts, file object, and config my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt}); + true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt, + strCompressType => $strCompressType}); # Create the stanza $oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_MISSING, @@ -192,7 +194,7 @@ sub run $oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1); $self->testResult( sub {storageRepo()->list(STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-2/0000000100000001')}, - '000000010000000100000001-' . $self->walGenerateContentChecksum(PG_VERSION_94) . '.' . COMPRESS_EXT, + '000000010000000100000001-' . $self->walGenerateContentChecksum(PG_VERSION_94) . ".${strCompressType}", 'check that WAL is in the archive at -2'); # Create the tablespace directory and perform a backup diff --git a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm index 8a388dec5..91a79f08e 100644 --- a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm @@ -62,15 +62,15 @@ sub run foreach my $strBackupDestination ( $bS3 || $bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER)) { - my $bCompress = $bHostBackup && !$bHostStandby; - my $bRepoEncrypt = ($bCompress && !$bS3) ? true : false; + my $strCompressType = $bHostBackup && !$bHostStandby ? GZ : NONE; + my $bRepoEncrypt = ($strCompressType ne NONE && !$bS3) ? true : false; # Increment the run, log, and decide whether this unit test should be run my $hyVm = vmGet(); my $strDbVersionMostRecent = ${$hyVm->{$self->vm()}{&VM_DB_TEST}}[-1]; next if (!$self->begin( - "bkp ${bHostBackup}, sby ${bHostStandby}, dst ${strBackupDestination}, cmp ${bCompress}, s3 ${bS3}, " . + "bkp ${bHostBackup}, sby ${bHostStandby}, dst ${strBackupDestination}, cmp ${strCompressType}, s3 ${bS3}, " . "enc ${bRepoEncrypt}", # Use the most recent db version on the expect vm for expect testing $self->vm() eq VM_EXPECT && $self->pgVersion() eq $strDbVersionMostRecent)); @@ -100,7 +100,7 @@ sub run my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( false, $self->expect(), {bHostBackup => $bHostBackup, bStandby => $bHostStandby, strBackupDestination => $strBackupDestination, - bCompress => $bCompress, bArchiveAsync => false, bS3 => $bS3, bRepoEncrypt => $bRepoEncrypt}); + strCompressType => $strCompressType, bArchiveAsync => false, bS3 => $bS3, bRepoEncrypt => $bRepoEncrypt}); # Only perform extra tests on certain runs to save time my $bTestLocal = $self->runCurrent() == 1; @@ -416,7 +416,7 @@ sub run # Kick out a bunch of archive logs to exercise async archiving. Only do this when compressed and remote to slow it # down enough to make it evident that the async process is working. - if ($bTestExtra && $bCompress && $strBackupDestination eq HOST_BACKUP) + if ($bTestExtra && $strCompressType ne NONE && $strBackupDestination eq HOST_BACKUP) { &log(INFO, ' multiple wal switches to exercise async archiving'); $oHostDbMaster->sqlExecute("create table wal_activity (id int)"); diff --git a/test/src/common/harnessStorage.c b/test/src/common/harnessStorage.c index 7ed289f4c..920bfea66 100644 --- a/test/src/common/harnessStorage.c +++ b/test/src/common/harnessStorage.c @@ -4,7 +4,7 @@ Storage Test Harness #include #include "common/debug.h" -#include "common/compress/gz/decompress.h" +#include "common/compress/helper.h" #include "common/user.h" #include "storage/storage.h" @@ -31,7 +31,8 @@ hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info) { uint64_t size = info->size; - // If the file is compressed then decompress to get the real size + // If the file is compressed then decompress to get the real size. Note that only gz is used in unit tests since + // it is the only compression type guaranteed to be present. if (data->fileCompressed) { ASSERT(data->storage != NULL); @@ -39,7 +40,7 @@ hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info) StorageRead *read = storageNewReadP( data->storage, data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name); - ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzDecompressNew()); + ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), decompressFilter(compressTypeGz)); size = bufUsed(storageGetP(read)); } diff --git a/test/src/module/command/archiveGetTest.c b/test/src/module/command/archiveGetTest.c index ad5af5dd1..8f5790601 100644 --- a/test/src/module/command/archiveGetTest.c +++ b/test/src/module/command/archiveGetTest.c @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Test Archive Get Command ***********************************************************************************************************************************/ -#include "common/compress/gz/compress.h" +#include "common/compress/helper.h" #include "common/harnessConfig.h" #include "common/harnessFork.h" #include "common/io/bufferRead.h" @@ -202,7 +202,7 @@ testRun(void) "repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz")); IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(destination)); - ioFilterGroupAdd(filterGroup, gzCompressNew(3)); + ioFilterGroupAdd(filterGroup, compressFilter(compressTypeGz, 3)); ioFilterGroupAdd( filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("worstpassphraseever"), NULL)); storagePutP(destination, buffer); diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index 1b194c4f6..97f2ed7c6 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -52,17 +52,19 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info) const String *manifestName = info->name; // If the file is compressed then decompress to get the real size - if (strEndsWithZ(info->name, "." GZ_EXT)) + CompressType compressType = compressTypeFromName(info->name); + + if (compressType != compressTypeNone) { ASSERT(data->storage != NULL); StorageRead *read = storageNewReadP( data->storage, data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name); - ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzDecompressNew()); + ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), decompressFilter(compressType)); size = bufUsed(storageGetP(read)); - manifestName = strSubN(info->name, 0, strSize(info->name) - strlen("." GZ_EXT)); + manifestName = strSubN(info->name, 0, strSize(info->name) - strSize(compressExtStr(compressType))); } strCatFmt(data->content, ", s=%" PRIu64, size); @@ -170,7 +172,7 @@ typedef struct TestBackupPqScriptParam bool backupStandby; bool errorAfterStart; bool noWal; // Don't write test WAL segments - bool walCompress; // Compress the archive files + CompressType walCompressType; // Compress type for the archive files unsigned int walTotal; // Total WAL to write unsigned int timeline; // Timeline to use for WAL files } TestBackupPqScriptParam; @@ -223,10 +225,10 @@ testBackupPqScript(unsigned int pgVersion, time_t backupTimeStart, TestBackupPqS storageRepoWrite(), strNewFmt( STORAGE_REPO_ARCHIVE "/%s/%s-%s%s", strPtr(archiveId), strPtr(strLstGet(walSegmentList, walSegmentIdx)), - strPtr(walChecksum), param.walCompress ? "." GZ_EXT : "")); + strPtr(walChecksum), strPtr(compressExtStr(param.walCompressType)))); - if (param.walCompress) - ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), gzCompressNew(1)); + if (param.walCompressType != compressTypeNone) + ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), compressFilter(param.walCompressType, 1)); storagePutP(write, walBuffer); } @@ -438,8 +440,8 @@ testRun(void) varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit varLstAdd(paramList, varNewStr(missingFile)); // repoFile varLstAdd(paramList, varNewBool(false)); // repoFileHasReference - varLstAdd(paramList, varNewBool(false)); // repoFileCompress - varLstAdd(paramList, varNewUInt(0)); // repoFileCompressLevel + varLstAdd(paramList, varNewUInt(compressTypeNone)); // repoFileCompress + varLstAdd(paramList, varNewInt(0)); // repoFileCompressLevel varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel varLstAdd(paramList, varNewBool(false)); // delta varLstAdd(paramList, NULL); // cipherSubPass @@ -513,8 +515,8 @@ testRun(void) varLstAdd(paramList, varNewUInt64(0xFFFFFFFFFFFFFFFF)); // pgFileChecksumPageLsnLimit varLstAdd(paramList, varNewStr(pgFile)); // repoFile varLstAdd(paramList, varNewBool(false)); // repoFileHasReference - varLstAdd(paramList, varNewBool(false)); // repoFileCompress - varLstAdd(paramList, varNewUInt(1)); // repoFileCompressLevel + varLstAdd(paramList, varNewUInt(compressTypeNone)); // repoFileCompress + varLstAdd(paramList, varNewInt(1)); // repoFileCompressLevel varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel varLstAdd(paramList, varNewBool(false)); // delta varLstAdd(paramList, NULL); // cipherSubPass @@ -555,8 +557,8 @@ testRun(void) varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit varLstAdd(paramList, varNewStr(pgFile)); // repoFile varLstAdd(paramList, varNewBool(true)); // repoFileHasReference - varLstAdd(paramList, varNewBool(false)); // repoFileCompress - varLstAdd(paramList, varNewUInt(1)); // repoFileCompressLevel + varLstAdd(paramList, varNewUInt(compressTypeNone)); // repoFileCompress + varLstAdd(paramList, varNewInt(1)); // repoFileCompressLevel varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel varLstAdd(paramList, varNewBool(true)); // delta varLstAdd(paramList, NULL); // cipherSubPass @@ -694,8 +696,8 @@ testRun(void) varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit varLstAdd(paramList, varNewStr(pgFile)); // repoFile varLstAdd(paramList, varNewBool(false)); // repoFileHasReference - varLstAdd(paramList, varNewBool(true)); // repoFileCompress - varLstAdd(paramList, varNewUInt(3)); // repoFileCompressLevel + varLstAdd(paramList, varNewUInt(compressTypeGz)); // repoFileCompress + varLstAdd(paramList, varNewInt(3)); // repoFileCompressLevel varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel varLstAdd(paramList, varNewBool(false)); // delta varLstAdd(paramList, NULL); // cipherSubPass @@ -812,8 +814,8 @@ testRun(void) varLstAdd(paramList, varNewUInt64(0)); // pgFileChecksumPageLsnLimit varLstAdd(paramList, varNewStr(pgFile)); // repoFile varLstAdd(paramList, varNewBool(false)); // repoFileHasReference - varLstAdd(paramList, varNewBool(false)); // repoFileCompress - varLstAdd(paramList, varNewUInt(0)); // repoFileCompressLevel + varLstAdd(paramList, varNewUInt(compressTypeNone)); // repoFileCompress + varLstAdd(paramList, varNewInt(0)); // repoFileCompressLevel varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel varLstAdd(paramList, varNewBool(false)); // delta varLstAdd(paramList, varNewStrZ("12345678")); // cipherPass @@ -1252,7 +1254,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("cannot resume when compression does not match"); - manifestResume->data.backupOptionCompress = true; + manifestResume->data.backupOptionCompressType = compressTypeGz; manifestSave( manifestResume, @@ -1264,12 +1266,12 @@ testRun(void) TEST_RESULT_LOG( "P00 WARN: backup '20191003-105320F' cannot be resumed:" - " new compression 'false' does not match resumable compression 'true'"); + " new compression 'none' does not match resumable compression 'gz'"); TEST_RESULT_BOOL( storagePathExistsP(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/20191003-105320F")), false, "check backup path removed"); - manifestResume->data.backupOptionCompress = false; + manifestResume->data.backupOptionCompressType = compressTypeNone; } // ***************************************************************************************************************************** @@ -1409,7 +1411,7 @@ testRun(void) TEST_RESULT_LOG( "P00 INFO: last backup label = [FULL-1], version = " PROJECT_VERSION "\n" - "P00 WARN: diff backup cannot alter compress option to 'true', reset to value in [FULL-1]\n" + "P00 WARN: diff backup cannot alter compress-type option to 'gz', reset to value in [FULL-1]\n" "P00 WARN: diff backup cannot alter hardlink option to 'true', reset to value in [FULL-1]"); // ------------------------------------------------------------------------------------------------------------------------- @@ -1613,7 +1615,7 @@ testRun(void) ManifestData *manifestResumeData = (ManifestData *)manifestData(manifestResume); manifestResumeData->backupType = backupTypeFull; - manifestResumeData->backupOptionCompress = true; + manifestResumeData->backupOptionCompressType = compressTypeGz; const String *resumeLabel = backupLabelCreate(backupTypeFull, NULL, backupTimeStart); manifestBackupLabelSet(manifestResume, resumeLabel); @@ -1669,6 +1671,12 @@ testRun(void) storagePathCreateP(storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/bogus_path", strPtr(resumeLabel))); // File is not in manifest + storagePutP( + storageNewWriteP( + storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/bogus.gz", strPtr(resumeLabel))), + NULL); + + // File has incorrect compression type storagePutP( storageNewWriteP( storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/global/bogus", strPtr(resumeLabel))), @@ -1702,6 +1710,8 @@ testRun(void) "P00 WARN: resumable backup 20191003-105320F of same type exists -- remove invalid files and resume\n" "P00 DETAIL: remove path '{[path]}/repo/backup/test1/20191003-105320F/pg_data/bogus_path' from resumed backup\n" "P00 DETAIL: remove file '{[path]}/repo/backup/test1/20191003-105320F/pg_data/global/bogus' from resumed backup" + " (mismatched compression type)\n" + "P00 DETAIL: remove file '{[path]}/repo/backup/test1/20191003-105320F/pg_data/global/bogus.gz' from resumed backup" " (missing in manifest)\n" "P00 DETAIL: remove file '{[path]}/repo/backup/test1/20191003-105320F/pg_data/global/pg_control.gz' from resumed" " backup (no checksum in resumed manifest)\n" @@ -1761,6 +1771,7 @@ testRun(void) strLstAdd(argList, strNewFmt("--" CFGOPT_PG1_PATH "=%s", strPtr(pg1Path))); strLstAddZ(argList, "--" CFGOPT_REPO1_RETENTION_FULL "=1"); strLstAddZ(argList, "--" CFGOPT_TYPE "=" BACKUP_TYPE_DIFF); + strLstAddZ(argList, "--no-" CFGOPT_COMPRESS); strLstAddZ(argList, "--" CFGOPT_STOP_AUTO); strLstAddZ(argList, "--" CFGOPT_REPO1_HARDLINK); harnessCfgLoad(cfgCmdBackup, argList); @@ -1777,7 +1788,7 @@ testRun(void) manifestResumeData->backupType = backupTypeDiff; manifestResumeData->backupLabelPrior = manifestData(manifestPrior)->backupLabel; - manifestResumeData->backupOptionCompress = true; + manifestResumeData->backupOptionCompressType = compressTypeGz; const String *resumeLabel = backupLabelCreate(backupTypeDiff, manifestData(manifestPrior)->backupLabel, backupTimeStart); manifestBackupLabelSet(manifestResume, resumeLabel); @@ -1834,6 +1845,7 @@ testRun(void) // Check log TEST_RESULT_LOG( "P00 INFO: last backup label = 20191003-105320F, version = " PROJECT_VERSION "\n" + "P00 WARN: diff backup cannot alter compress-type option to 'none', reset to value in 20191003-105320F\n" "P00 INFO: execute exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n" "P00 INFO: backup start archive = 0000000105D9759000000000, lsn = 5d97590/0\n" "P00 WARN: file 'time-mismatch2' has timestamp in the future, enabling delta checksum\n" @@ -1953,7 +1965,7 @@ testRun(void) storagePathRemoveP(storageRepoWrite(), STRDEF(STORAGE_REPO_BACKUP "/20191016-042640F"), .recurse = true); // Run backup - testBackupPqScriptP(PG_VERSION_96, backupTimeStart, .backupStandby = true, .walCompress = true); + testBackupPqScriptP(PG_VERSION_96, backupTimeStart, .backupStandby = true, .walCompressType = compressTypeGz); TEST_RESULT_VOID(cmdBackup(), "backup"); // Set log level back to detail @@ -2089,7 +2101,7 @@ testRun(void) ((Storage *)storageRepoWrite())->interface.feature ^= 1 << storageFeatureHardLink; // Run backup - testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .walCompress = true, .walTotal = 3); + testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .walCompressType = compressTypeGz, .walTotal = 3); TEST_RESULT_VOID(cmdBackup(), "backup"); // Reset storage features diff --git a/test/src/module/command/helpTest.c b/test/src/module/command/helpTest.c index 8e59cb405..9988d6403 100644 --- a/test/src/module/command/helpTest.c +++ b/test/src/module/command/helpTest.c @@ -149,8 +149,7 @@ testRun(void) " --buffer-size buffer size for file operations\n" " [current=32768, default=4194304]\n" " --cmd-ssh path to ssh client executable [default=ssh]\n" - " --compress-level-network compression level for network transfer when\n" - " compress=n [default=3]\n" + " --compress-level-network network compression level [default=3]\n" " --config pgBackRest configuration file\n" " [default=/etc/pgbackrest/pgbackrest.conf]\n" " --config-include-path path to additional pgBackRest configuration\n" diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index a602ab6c7..8f4686981 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Test Restore Command ***********************************************************************************************************************************/ -#include "common/compress/gz/compress.h" +#include "common/compress/helper.h" #include "common/crypto/cipherBlock.h" #include "common/io/io.h" #include "common/io/bufferRead.h" @@ -196,7 +196,7 @@ testRun(void) StorageWrite *ceRepoFile = storageNewWriteP( storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strPtr(repoFileReferenceFull), strPtr(repoFile1))); IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(ceRepoFile)); - ioFilterGroupAdd(filterGroup, gzCompressNew(3)); + ioFilterGroupAdd(filterGroup, compressFilter(compressTypeGz, 3)); ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("badpass"), NULL)); storagePutP(ceRepoFile, BUFSTRDEF("acefile")); @@ -316,7 +316,7 @@ testRun(void) VariantList *paramList = varLstNew(); varLstAdd(paramList, varNewStr(repoFile1)); varLstAdd(paramList, varNewStr(repoFileReferenceFull)); - varLstAdd(paramList, varNewBool(false)); + varLstAdd(paramList, varNewUInt(compressTypeNone)); varLstAdd(paramList, varNewStrZ("protocol")); varLstAdd(paramList, varNewStrZ("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")); varLstAdd(paramList, varNewBool(false)); @@ -347,7 +347,7 @@ testRun(void) paramList = varLstNew(); varLstAdd(paramList, varNewStr(repoFile1)); varLstAdd(paramList, varNewStr(repoFileReferenceFull)); - varLstAdd(paramList, varNewBool(false)); + varLstAdd(paramList, varNewUInt(compressTypeNone)); varLstAdd(paramList, varNewStrZ("protocol")); varLstAdd(paramList, varNewStrZ("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67")); varLstAdd(paramList, varNewBool(false)); diff --git a/test/src/module/common/compressTest.c b/test/src/module/common/compressTest.c index 80275592a..6a5493976 100644 --- a/test/src/module/common/compressTest.c +++ b/test/src/module/common/compressTest.c @@ -5,6 +5,7 @@ Test Compression #include "common/io/bufferRead.h" #include "common/io/bufferWrite.h" #include "common/io/io.h" +#include "storage/posix/storage.h" /*********************************************************************************************************************************** Compress data @@ -35,7 +36,7 @@ testCompress(IoFilter *compress, Buffer *decompressed, size_t inputSize, size_t } ioWriteClose(write); - memContextFree(((GzCompress *)ioFilterDriver(compress))->memContext); + ioFilterFree(compress); return compressed; } @@ -63,11 +64,107 @@ testDecompress(IoFilter *decompress, Buffer *compressed, size_t inputSize, size_ ioReadClose(read); bufFree(output); - memContextFree(((GzDecompress *)ioFilterDriver(decompress))->memContext); + ioFilterFree(decompress); return decompressed; } +/*********************************************************************************************************************************** +Standard test suite to be applied to all compression types +***********************************************************************************************************************************/ +static void +testSuite(CompressType type, const char *decompressCmd) +{ + const char *simpleData = "A simple string"; + Buffer *compressed = NULL; + Buffer *decompressed = bufNewC(simpleData, strlen(simpleData)); + + VariantList *compressParamList = varLstNew(); + varLstAdd(compressParamList, varNewUInt(1)); + + // Create default storage object for testing + Storage *storageTest = storagePosixNew(strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); + + TEST_TITLE("simple data"); + + TEST_ASSIGN( + compressed, + testCompress( + compressFilterVar(strNewFmt("%sCompress", strPtr(compressTypeStr(type))), compressParamList), decompressed, 1024, + 256 * 1024 * 1024), + "simple data - compress large in/large out buffer"); + + TEST_RESULT_BOOL( + bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1024, 1)), true, + "simple data - compress large in/small out buffer"); + + TEST_RESULT_BOOL( + bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1024)), true, + "simple data - compress small in/large out buffer"); + + TEST_RESULT_BOOL( + bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1)), true, + "simple data - compress small in/small out buffer"); + + TEST_RESULT_BOOL( + bufEq( + decompressed, + testDecompress( + compressFilterVar(strNewFmt("%sDecompress", strPtr(compressTypeStr(type))), NULL), compressed, 1024, 1024)), + true, "simple data - decompress large in/large out buffer"); + + TEST_RESULT_BOOL( + bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1024, 1)), true, + "simple data - decompress large in/small out buffer"); + + TEST_RESULT_BOOL( + bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1024)), true, + "simple data - decompress small in/large out buffer"); + + TEST_RESULT_BOOL( + bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1)), true, + "simple data - decompress small in/small out buffer"); + + // ------------------------------------------------------------------------------------------------------------------------- + if (decompressCmd != NULL) + { + TEST_TITLE("compressed output can be decompressed with command-line tool"); + + storagePutP(storageNewWriteP(storageTest, STRDEF("test.cmp")), compressed); + TEST_SYSTEM_FMT("%s {[path]}/test.cmp > {[path]}/test.out", decompressCmd); + TEST_RESULT_BOOL(bufEq(decompressed, storageGetP(storageNewReadP(storageTest, STRDEF("test.out")))), true, "check output"); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on no compression data"); + + TEST_ERROR(testDecompress(decompressFilter(type), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on truncated compression data"); + + Buffer *truncated = bufNew(0); + bufCatSub(truncated, compressed, 0, bufUsed(compressed) - 1); + + TEST_RESULT_UINT(bufUsed(truncated), bufUsed(compressed) - 1, "check truncated buffer size"); + TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compress a large zero input buffer into small output buffer"); + + decompressed = bufNew(1024 * 1024 - 1); + memset(bufPtr(decompressed), 0, bufSize(decompressed)); + bufUsedSet(decompressed, bufSize(decompressed)); + + TEST_ASSIGN( + compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 1024), + "zero data - compress large in/small out buffer"); + + TEST_RESULT_BOOL( + bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true, + "zero data - decompress large in/small out buffer"); +} + /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -77,8 +174,14 @@ testRun(void) FUNCTION_HARNESS_VOID(); // ***************************************************************************************************************************** - if (testBegin("gzError")) + if (testBegin("gz")) { + // Run standard test suite + testSuite(compressTypeGz, "gzip -dc"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("gzError()"); + TEST_RESULT_INT(gzError(Z_OK), Z_OK, "check ok"); TEST_RESULT_INT(gzError(Z_STREAM_END), Z_STREAM_END, "check stream end"); TEST_ERROR(gzError(Z_NEED_DICT), AssertError, "zlib threw error: [2] need dictionary"); @@ -89,91 +192,73 @@ testRun(void) TEST_ERROR(gzError(Z_BUF_ERROR), AssertError, "zlib threw error: [-5] no space in buffer"); TEST_ERROR(gzError(Z_VERSION_ERROR), FormatError, "zlib threw error: [-6] incompatible version"); TEST_ERROR(gzError(999), AssertError, "zlib threw error: [999] unknown error"); - } - - // ***************************************************************************************************************************** - if (testBegin("GzCompress and GzDecompress")) - { - const char *simpleData = "A simple string"; - Buffer *compressed = NULL; - Buffer *decompressed = bufNewC(simpleData, strlen(simpleData)); - - VariantList *compressParamList = varLstNew(); - varLstAdd(compressParamList, varNewUInt(3)); - varLstAdd(compressParamList, varNewBool(false)); - - TEST_ASSIGN( - compressed, testCompress(gzCompressNewVar(compressParamList), decompressed, 1024, 1024), - "simple data - compress large in/large out buffer"); - - TEST_RESULT_BOOL( - bufEq(compressed, testCompress(gzCompressNew(3), decompressed, 1024, 1)), true, - "simple data - compress large in/small out buffer"); - - TEST_RESULT_BOOL( - bufEq(compressed, testCompress(gzCompressNew(3), decompressed, 1, 1024)), true, - "simple data - compress small in/large out buffer"); - - TEST_RESULT_BOOL( - bufEq(compressed, testCompress(gzCompressNew(3), decompressed, 1, 1)), true, - "simple data - compress small in/small out buffer"); - - TEST_RESULT_BOOL( - bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, 1024, 1024)), true, - "simple data - decompress large in/large out buffer"); - - TEST_RESULT_BOOL( - bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, 1024, 1)), true, - "simple data - decompress large in/small out buffer"); - - TEST_RESULT_BOOL( - bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, 1, 1024)), true, - "simple data - decompress small in/large out buffer"); - - TEST_RESULT_BOOL( - bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, 1, 1)), true, - "simple data - decompress small in/small out buffer"); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("error on no compression data"); + TEST_TITLE("gzDecompressToLog() and gzCompressToLog()"); - TEST_ERROR(testDecompress(gzDecompressNew(), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("error on truncated compression data"); - - Buffer *truncated = bufNew(0); - bufCatSub(truncated, compressed, 0, bufUsed(compressed) - 1); - - TEST_RESULT_UINT(bufUsed(truncated), bufUsed(compressed) - 1, "check truncated buffer size"); - TEST_ERROR(testDecompress(gzDecompressNew(), truncated, 512, 512), FormatError, "unexpected eof in compressed data"); - - // Compress a large zero input buffer into small output buffer - // ------------------------------------------------------------------------------------------------------------------------- - decompressed = bufNew(1024 * 1024 - 1); - memset(bufPtr(decompressed), 0, bufSize(decompressed)); - bufUsedSet(decompressed, bufSize(decompressed)); - - TEST_ASSIGN( - compressed, testCompress(gzCompressNew(3), decompressed, bufSize(decompressed), 1024), - "zero data - compress large in/small out buffer"); - - TEST_RESULT_BOOL( - bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, bufSize(compressed), 1024 * 256)), true, - "zero data - decompress large in/small out buffer"); - } - - // ***************************************************************************************************************************** - if (testBegin("gzDecompressToLog() and gzCompressToLog()")) - { GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew()); TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: false, done: false, availIn: 0}", "format object"); decompress->inputSame = true; decompress->done = true; + TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: true, done: true, availIn: 0}", "format object"); } + // Test everything in the helper that is not tested in the individual compression type tests + // ***************************************************************************************************************************** + if (testBegin("helper")) + { + TEST_TITLE("compressTypeEnum()"); + + TEST_RESULT_UINT(compressTypeEnum(STRDEF("none")), compressTypeNone, "none enum"); + TEST_RESULT_UINT(compressTypeEnum(STRDEF("gz")), compressTypeGz, "gz enum"); + TEST_ERROR(compressTypeEnum(strNew(BOGUS_STR)), AssertError, "invalid compression type 'BOGUS'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compressTypePresent()"); + + TEST_RESULT_VOID(compressTypePresent(compressTypeNone), "type none always present"); + TEST_ERROR(compressTypePresent(compressTypeZst), OptionInvalidValueError, "pgBackRest not compiled with zst support"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compressTypeFromName()"); + + TEST_RESULT_UINT(compressTypeFromName(STRDEF("file")), compressTypeNone, "type from name"); + TEST_RESULT_UINT(compressTypeFromName(STRDEF("file.gz")), compressTypeGz, "type from name"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compressFilterVar()"); + + TEST_RESULT_PTR(compressFilterVar(STRDEF("BOGUS"), 0), NULL, "no filter match"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compressExtStr()"); + + TEST_RESULT_STR_Z(compressExtStr(compressTypeNone), "", "one ext"); + TEST_RESULT_STR_Z(compressExtStr(compressTypeGz), ".gz", "gz ext"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compressExtCat()"); + + String *file = strNew("file"); + TEST_RESULT_VOID(compressExtCat(file, compressTypeGz), "cat gz ext"); + TEST_RESULT_STR_Z(file, "file.gz", " check gz ext"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compressExtStrip()"); + + TEST_ERROR(compressExtStrip(STRDEF("file"), compressTypeGz), FormatError, "'file' must have '.gz' extension"); + TEST_RESULT_STR_Z(compressExtStrip(STRDEF("file"), compressTypeNone), "file", "nothing to strip"); + TEST_RESULT_STR_Z(compressExtStrip(STRDEF("file.gz"), compressTypeGz), "file", "strip gz"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compressLevelDefault()"); + + TEST_RESULT_INT(compressLevelDefault(compressTypeNone), 0, "none level=0"); + TEST_RESULT_INT(compressLevelDefault(compressTypeGz), 6, "gz level=6"); + } + FUNCTION_HARNESS_RESULT_VOID(); } diff --git a/test/src/module/config/loadTest.c b/test/src/module/config/loadTest.c index 424f231a4..4f46c4f9c 100644 --- a/test/src/module/config/loadTest.c +++ b/test/src/module/config/loadTest.c @@ -264,9 +264,52 @@ testRun(void) TEST_RESULT_VOID(harnessCfgLoad(cfgCmdArchiveGet, argList), "valid bucket name"); TEST_RESULT_STR_Z(cfgOptionStr(cfgOptRepoS3Bucket), "cool-bucket", " check bucket value"); + TEST_RESULT_BOOL(cfgOptionValid(cfgOptCompress), false, " compress is not valid"); unsetenv("PGBACKREST_REPO1_S3_KEY"); unsetenv("PGBACKREST_REPO1_S3_KEY_SECRET"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compress-type=none when compress=n"); + + argList = strLstNew(); + strLstAddZ(argList, "--" CFGOPT_STANZA "=db"); + strLstAddZ(argList, "--no-" CFGOPT_COMPRESS); + + TEST_RESULT_VOID(harnessCfgLoad(cfgCmdArchivePush, argList), "load config"); + TEST_RESULT_STR_Z(cfgOptionStr(cfgOptCompressType), "none", " compress-type=none"); + TEST_RESULT_INT(cfgOptionInt(cfgOptCompressLevel), 0, " compress-level=0"); + TEST_RESULT_BOOL(cfgOptionValid(cfgOptCompress), false, " compress is not valid"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("compress-type=gz when compress=y"); + + argList = strLstNew(); + strLstAddZ(argList, "--" CFGOPT_STANZA "=db"); + strLstAddZ(argList, "--" CFGOPT_COMPRESS); + strLstAddZ(argList, "--" CFGOPT_COMPRESS_LEVEL "=9"); + + TEST_RESULT_VOID(harnessCfgLoad(cfgCmdArchivePush, argList), "load config"); + TEST_RESULT_STR_Z(cfgOptionStr(cfgOptCompressType), "gz", " compress-type=gz"); + TEST_RESULT_INT(cfgOptionInt(cfgOptCompressLevel), 9, " compress-level=9"); + TEST_RESULT_BOOL(cfgOptionValid(cfgOptCompress), false, " compress is not valid"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("warn when compress-type and compress both set"); + + argList = strLstNew(); + strLstAddZ(argList, "--" CFGOPT_STANZA "=db"); + strLstAddZ(argList, "--no-" CFGOPT_COMPRESS); + strLstAddZ(argList, "--" CFGOPT_COMPRESS_TYPE "=gz"); + + TEST_RESULT_VOID(harnessCfgLoad(cfgCmdArchivePush, argList), "load config"); + TEST_RESULT_STR_Z(cfgOptionStr(cfgOptCompressType), "gz", " compress-type=gz"); + TEST_RESULT_INT(cfgOptionInt(cfgOptCompressLevel), 6, " compress-level=6"); + TEST_RESULT_BOOL(cfgOptionValid(cfgOptCompress), false, " compress is not valid"); + + harnessLogResult( + "P00 WARN: 'compress' and 'compress-type' options should not both be set\n" + " HINT: 'compress-type' is preferred and 'compress' is deprecated."); } // ***************************************************************************************************************************** diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index 9d9451522..ab23f0e22 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -115,6 +115,7 @@ testRun(void) "option-archive-copy=false\n" \ "option-checksum-page=false\n" \ "option-compress=false\n" \ + "option-compress-type=\"none\"\n" \ "option-hardlink=false\n" \ "option-online=false\n" @@ -132,11 +133,13 @@ testRun(void) #define TEST_MANIFEST_OPTION_ONLINE_FALSE \ "option-compress=false\n" \ + "option-compress-type=\"none\"\n" \ "option-hardlink=false\n" \ "option-online=false\n" #define TEST_MANIFEST_OPTION_ONLINE_TRUE \ "option-compress=false\n" \ + "option-compress-type=\"none\"\n" \ "option-hardlink=false\n" \ "option-online=true\n" @@ -1173,7 +1176,7 @@ testRun(void) TEST_RESULT_VOID(manifestBuildValidate(manifest, true, 1482182860, false), "validate manifest"); TEST_RESULT_UINT(manifest->data.backupTimestampCopyStart, 1482182861, "check copy start"); TEST_RESULT_BOOL(varBool(manifest->data.backupOptionDelta), true, "check delta"); - TEST_RESULT_BOOL(manifest->data.backupOptionCompress, false, "check compress"); + TEST_RESULT_UINT(manifest->data.backupOptionCompressType, compressTypeNone, "check compress"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("timestamp in past does not force delta"); @@ -1194,7 +1197,7 @@ testRun(void) TEST_RESULT_VOID(manifestBuildValidate(manifest, false, 1482182859, true), "validate manifest"); TEST_RESULT_UINT(manifest->data.backupTimestampCopyStart, 1482182859, "check copy start"); TEST_RESULT_BOOL(varBool(manifest->data.backupOptionDelta), true, "check delta"); - TEST_RESULT_BOOL(manifest->data.backupOptionCompress, true, "check compress"); + TEST_RESULT_UINT(manifest->data.backupOptionCompressType, compressTypeGz, "check compress"); TEST_RESULT_LOG("P00 WARN: file 'PG_VERSION' has timestamp in the future, enabling delta checksum"); } @@ -1221,7 +1224,8 @@ testRun(void) "[backup:option]\n" \ "option-archive-check=false\n" \ "option-archive-copy=false\n" \ - "option-compress=false\n" + "option-compress=false\n" \ + "option-compress-type=\"none\"\n" #define TEST_MANIFEST_HEADER_POST \ "option-hardlink=false\n" \ @@ -1566,6 +1570,7 @@ testRun(void) "option-archive-check=true\n" "option-archive-copy=true\n" "option-compress=false\n" + "option-compress-type=\"none\"\n" "option-hardlink=false\n" "option-online=false\n" "\n" @@ -1651,6 +1656,7 @@ testRun(void) "option-compress=true\n" \ "option-compress-level=3\n" \ "option-compress-level-network=6\n" \ + "option-compress-type=\"gz\"\n" \ "option-delta=false\n" \ "option-hardlink=true\n" \ "option-online=false\n" \ diff --git a/test/src/module/storage/remoteTest.c b/test/src/module/storage/remoteTest.c index 135d0be99..e7c866ebf 100644 --- a/test/src/module/storage/remoteTest.c +++ b/test/src/module/storage/remoteTest.c @@ -447,8 +447,8 @@ testRun(void) ioFilterGroupAdd(filterGroup, pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, 0)); ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL)); ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL)); - ioFilterGroupAdd(filterGroup, gzCompressNew(3)); - ioFilterGroupAdd(filterGroup, gzDecompressNew()); + ioFilterGroupAdd(filterGroup, compressFilter(compressTypeGz, 3)); + ioFilterGroupAdd(filterGroup, decompressFilter(compressTypeGz)); varLstAdd(paramList, ioFilterGroupParamAll(filterGroup)); TEST_RESULT_BOOL(