From 794c5771305bfce6298ee529df682d3a873f6f7d Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 6 Mar 2024 11:00:09 +1300 Subject: [PATCH] Migrate integration tests to C. The Perl integration tests were migrated as faithfully as possible, but there was some cruft and a few unit tests that it did not make sense to migrate. Also remove all Perl code made obsolete by this migration. All unit, performance, and integration tests are now written in C but significant parts of the test harness remain to be migrated. --- .github/workflows/test.yml | 6 +- doc/lib/pgBackRestDoc/Common/DocExecute.pm | 6 +- .../lib/pgBackRestDoc/Common/Host.pm | 4 +- .../lib/pgBackRestDoc/Common/HostGroup.pm | 6 +- src/build/common/exec.c | 20 +- src/build/common/exec.h | 1 + test/certificate/pgbackrest-test-server.cnf | 6 +- test/certificate/pgbackrest-test-server.crt | 33 +- test/ci.pl | 4 +- test/define.yaml | 12 +- .../lib/pgBackRestTest/Common/CoverageTest.pm | 26 +- test/lib/pgBackRestTest/Common/FileTest.pm | 277 -- test/lib/pgBackRestTest/Common/JobTest.pm | 77 +- test/lib/pgBackRestTest/Common/RunTest.pm | 426 --- test/lib/pgBackRestTest/Common/StorageRepo.pm | 623 ----- test/lib/pgBackRestTest/Env/ArchiveInfo.pm | 478 ---- test/lib/pgBackRestTest/Env/BackupInfo.pm | 898 ------- test/lib/pgBackRestTest/Env/ExpireEnvTest.pm | 639 ----- .../pgBackRestTest/Env/Host/HostAzureTest.pm | 82 - .../pgBackRestTest/Env/Host/HostBackupTest.pm | 2321 ----------------- .../pgBackRestTest/Env/Host/HostBaseTest.pm | 139 - .../Env/Host/HostDbCommonTest.pm | 217 -- .../Env/Host/HostDbSyntheticTest.pm | 717 ----- .../lib/pgBackRestTest/Env/Host/HostDbTest.pm | 528 ---- .../pgBackRestTest/Env/Host/HostGcsTest.pm | 80 - .../lib/pgBackRestTest/Env/Host/HostS3Test.pm | 86 - .../pgBackRestTest/Env/Host/HostSftpTest.pm | 72 - test/lib/pgBackRestTest/Env/HostEnvTest.pm | 543 ---- test/lib/pgBackRestTest/Env/InfoCommon.pm | 32 - test/lib/pgBackRestTest/Env/Manifest.pm | 1446 ---------- .../pgBackRestTest/Module/Real/RealAllTest.pm | 747 ------ test/src/build/config/config.yaml | 6 + test/src/build/help/help.xml | 10 + test/src/command/test/build.c | 32 +- test/src/command/test/build.h | 22 +- test/src/command/test/define.c | 89 +- test/src/command/test/define.h | 2 + test/src/command/test/test.c | 19 +- test/src/command/test/test.h | 6 +- test/src/common/harnessHost.c | 1301 +++++++++ test/src/common/harnessHost.h | 490 ++++ test/src/common/harnessTest.c | 35 +- test/src/common/harnessTest.h | 17 + test/src/common/harnessTest.intern.h | 3 +- test/src/main.c | 2 +- test/src/module/common/execTest.c | 9 +- test/src/module/integration/allTest.c | 404 +++ test/src/module/test/testTest.c | 174 +- test/src/test.c | 9 + test/src/valgrind.suppress.none | 16 + test/test.pl | 1375 +++++----- 51 files changed, 3348 insertions(+), 11225 deletions(-) rename test/lib/pgBackRestTest/Common/HostTest.pm => doc/lib/pgBackRestDoc/Common/Host.pm (99%) rename test/lib/pgBackRestTest/Common/HostGroupTest.pm => doc/lib/pgBackRestDoc/Common/HostGroup.pm (96%) delete mode 100644 test/lib/pgBackRestTest/Common/FileTest.pm delete mode 100644 test/lib/pgBackRestTest/Common/RunTest.pm delete mode 100644 test/lib/pgBackRestTest/Common/StorageRepo.pm delete mode 100644 test/lib/pgBackRestTest/Env/ArchiveInfo.pm delete mode 100644 test/lib/pgBackRestTest/Env/BackupInfo.pm delete mode 100644 test/lib/pgBackRestTest/Env/ExpireEnvTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/Host/HostDbTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/Host/HostS3Test.pm delete mode 100644 test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/HostEnvTest.pm delete mode 100644 test/lib/pgBackRestTest/Env/InfoCommon.pm delete mode 100644 test/lib/pgBackRestTest/Env/Manifest.pm delete mode 100644 test/lib/pgBackRestTest/Module/Real/RealAllTest.pm create mode 100644 test/src/common/harnessHost.c create mode 100644 test/src/common/harnessHost.h create mode 100644 test/src/module/integration/allTest.c create mode 100644 test/src/valgrind.suppress.none diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d27fabdb..cc9de1b70 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,13 +32,13 @@ jobs: - param: test --vm=d10 --param=no-performance --param=c-only # All integration tests for 32-bit - - param: test --vm=d10 --param=module=mock --param=module=real + - param: test --vm=d10 --param=module=mock --param=module=integration # Debian/Ubuntu documentation - param: doc --vm=u20 # All integration tests - - param: test --vm=u22 --param=build-package --param=module=mock --param=module=real + - param: test --vm=u22 --param=build-package --param=module=mock --param=module=integration # All unit tests with coverage, backtrace and alternate timezone - param: test --vm=u22 --param=c-only --param=no-valgrind --param=tz=America/New_York @@ -53,7 +53,7 @@ jobs: - param: doc --vm=rh8 # All integration tests - - param: test --vm=rh7 --param=module=mock --param=module=real + - param: test --vm=rh7 --param=module=mock --param=module=integration steps: - name: Checkout Code diff --git a/doc/lib/pgBackRestDoc/Common/DocExecute.pm b/doc/lib/pgBackRestDoc/Common/DocExecute.pm index 91f360b78..695166d34 100644 --- a/doc/lib/pgBackRestDoc/Common/DocExecute.pm +++ b/doc/lib/pgBackRestDoc/Common/DocExecute.pm @@ -16,11 +16,11 @@ use File::Basename qw(dirname); use Storable qw(dclone); use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostTest; -use pgBackRestTest::Common::HostGroupTest; use pgBackRestDoc::Common::DocManifest; use pgBackRestDoc::Common::Exception; +use pgBackRestDoc::Common::Host; +use pgBackRestDoc::Common::HostGroup; use pgBackRestDoc::Common::Ini; use pgBackRestDoc::Common::Log; use pgBackRestDoc::Common::String; @@ -1050,7 +1050,7 @@ sub sectionChildProcess $strOption =~ s/\{\[host\-repo\-path\]\}/${strHostRepoPath}/g; } - my $oHost = new pgBackRestTest::Common::HostTest( + my $oHost = new pgBackRestDoc::Common::Host( $$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage, $strHostUser, defined($strMount) ? [$strMount] : undef, $strOption, $$hCacheKey{param}, $$hCacheKey{'update-hosts'}); diff --git a/test/lib/pgBackRestTest/Common/HostTest.pm b/doc/lib/pgBackRestDoc/Common/Host.pm similarity index 99% rename from test/lib/pgBackRestTest/Common/HostTest.pm rename to doc/lib/pgBackRestDoc/Common/Host.pm index d7d76a4e0..20606976b 100644 --- a/test/lib/pgBackRestTest/Common/HostTest.pm +++ b/doc/lib/pgBackRestDoc/Common/Host.pm @@ -1,7 +1,7 @@ #################################################################################################################################### -# HostTest.pm - Encapsulate a docker host for testing +# HostTest.pm - Encapsulate a docker host #################################################################################################################################### -package pgBackRestTest::Common::HostTest; +package pgBackRestDoc::Common::Host; #################################################################################################################################### # Perl includes diff --git a/test/lib/pgBackRestTest/Common/HostGroupTest.pm b/doc/lib/pgBackRestDoc/Common/HostGroup.pm similarity index 96% rename from test/lib/pgBackRestTest/Common/HostGroupTest.pm rename to doc/lib/pgBackRestDoc/Common/HostGroup.pm index 345a54e04..e49bc4463 100644 --- a/test/lib/pgBackRestTest/Common/HostGroupTest.pm +++ b/doc/lib/pgBackRestDoc/Common/HostGroup.pm @@ -1,7 +1,7 @@ #################################################################################################################################### -# HostGroupTest.pm - Encapsulate a group of docker containers for testing +# HostGroupTest.pm - Encapsulate a group of docker containers #################################################################################################################################### -package pgBackRestTest::Common::HostGroupTest; +package pgBackRestDoc::Common::HostGroup; #################################################################################################################################### # Perl includes @@ -177,7 +177,7 @@ sub hostGroupGet { if (!defined($oHostGroup)) { - $oHostGroup = new pgBackRestTest::Common::HostGroupTest(); + $oHostGroup = new pgBackRestDoc::Common::HostGroup(); } return $oHostGroup; diff --git a/src/build/common/exec.c b/src/build/common/exec.c index ebe27d195..eb1e52d29 100644 --- a/src/build/common/exec.c +++ b/src/build/common/exec.c @@ -9,10 +9,11 @@ Execute Process Extensions /**********************************************************************************************************************************/ static String * -execProcess(Exec *const this) +execProcess(Exec *const this, const ExecOneParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(EXEC, this); + FUNCTION_LOG_PARAM(INT, param.resultExpect); FUNCTION_LOG_END(); String *const result = strNew(); @@ -35,7 +36,7 @@ execProcess(Exec *const this) // If the process exited normally but without a success status if (WIFEXITED(processStatus)) { - if (WEXITSTATUS(processStatus) != 0) + if (WEXITSTATUS(processStatus) != param.resultExpect) execCheckStatusError(this, processStatus, strTrim(result)); } // Else if the process did not exit normally then it must have been a signal @@ -53,7 +54,8 @@ execOne(const String *const command, const ExecOneParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, command); - (void)param; + FUNCTION_LOG_PARAM(STRING, param.shell); + FUNCTION_LOG_PARAM(INT, param.resultExpect); FUNCTION_LOG_END(); String *result = NULL; @@ -61,23 +63,23 @@ execOne(const String *const command, const ExecOneParam param) MEM_CONTEXT_TEMP_BEGIN() { const StringList *const shellList = strLstNewSplitZ(param.shell != NULL ? param.shell : STRDEF("sh -c"), " "); - StringList *const param = strLstNew(); + StringList *const paramList = strLstNew(); ASSERT(strLstSize(shellList) != 0); for (unsigned int shellIdx = 1; shellIdx < strLstSize(shellList); shellIdx++) - strLstAdd(param, strLstGet(shellList, shellIdx)); + strLstAdd(paramList, strLstGet(shellList, shellIdx)); - strLstAddFmt(param, "%s 2>&1", strZ(command)); - strLstAddZ(param, "2>&1"); + strLstAddFmt(paramList, "%s 2>&1", strZ(command)); + strLstAddZ(paramList, "2>&1"); - Exec *const exec = execNew(strLstGet(shellList, 0), param, command, ioTimeoutMs()); + Exec *const exec = execNew(strLstGet(shellList, 0), paramList, command, ioTimeoutMs()); execOpen(exec); MEM_CONTEXT_PRIOR_BEGIN() { - result = execProcess(exec); + result = execProcess(exec, param); } MEM_CONTEXT_PRIOR_END(); } diff --git a/src/build/common/exec.h b/src/build/common/exec.h index cf53a4377..94171dcd0 100644 --- a/src/build/common/exec.h +++ b/src/build/common/exec.h @@ -17,6 +17,7 @@ typedef struct ExecOneParam { VAR_PARAM_HEADER; const String *shell; // Shell command to use for exec (default is sh -c) + int resultExpect; // Expected result, if not 0 } ExecOneParam; #define execOneP(command, ...) \ diff --git a/test/certificate/pgbackrest-test-server.cnf b/test/certificate/pgbackrest-test-server.cnf index 034909753..7443b43b5 100644 --- a/test/certificate/pgbackrest-test-server.cnf +++ b/test/certificate/pgbackrest-test-server.cnf @@ -37,6 +37,6 @@ DNS.4 = 127.0.0.1 IP.1 = 127.0.0.1 # Used in integration tests -DNS.5 = db-primary -DNS.6 = db-standby -DNS.7 = backup +DNS.5 = pg1 +DNS.6 = pg2 +DNS.7 = repo diff --git a/test/certificate/pgbackrest-test-server.crt b/test/certificate/pgbackrest-test-server.crt index 95040f2be..01c904265 100644 --- a/test/certificate/pgbackrest-test-server.crt +++ b/test/certificate/pgbackrest-test-server.crt @@ -1,8 +1,8 @@ -----BEGIN CERTIFICATE----- -MIIGAjCCA+qgAwIBAgIUW0gPWoZD5DqjIWIP3PliYA0IAOQwDQYJKoZIhvcNAQEL +MIIF8jCCA9qgAwIBAgIUJCya0E5vFzyH2AgiM3HSAHmpZ1QwDQYJKoZIhvcNAQEL BQAwXDELMAkGA1UEBhMCVVMxDDAKBgNVBAgMA0FsbDEMMAoGA1UEBwwDQWxsMRMw EQYDVQQKDApwZ0JhY2tSZXN0MRwwGgYDVQQDDBN0ZXN0LnBnYmFja3Jlc3Qub3Jn -MCAXDTIxMDgyNjEyMjkwM1oYDzIyOTUwNjEwMTIyOTAzWjB6MQswCQYDVQQGEwJV +MCAXDTI0MDMwNDAxMzgzMFoYDzIyOTcxMjE3MDEzODMwWjB6MQswCQYDVQQGEwJV UzEMMAoGA1UECAwDQWxsMQwwCgYDVQQHDANBbGwxEzARBgNVBAoMCnBnQmFja1Jl c3QxHDAaBgNVBAsME1VuaXQgVGVzdGluZyBEb21haW4xHDAaBgNVBAMME3Rlc3Qu cGdiYWNrcmVzdC5vcmcwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDD @@ -16,20 +16,19 @@ dCdyXuSftGFx0JxvmDhl9qFGarv1BKgwO83j7sy3IREte1K21JaIHNBVWP+NwU0N 4Z4OMqnpnnnGiyi0xnfJVqOXghu5BLWl9MuOntZ0nnzLmFD7w795uNRgjE6jmRmF FlX+PGqhHsZr0wZsBDsE9xO4i2l8aqJZx1hT5l3LIXC+lei/qo2gJi3nyePuz4UB t53sTNEdrZndFUaRyq/aJfkR13J0eaoqKn5BRRHhw8tRef6S84e0kQ6ABYNRGHQN -V+GswPl1fV37114FTBnz2Bi/GSQSs8vWjw49HHKK5wIDAQABo4GbMIGYMAkGA1Ud -EwQCMAAwCwYDVR0PBAQDAgXgMH4GA1UdEQR3MHWCE3Rlc3QucGdiYWNrcmVzdC5v +V+GswPl1fV37114FTBnz2Bi/GSQSs8vWjw49HHKK5wIDAQABo4GLMIGIMAkGA1Ud +EwQCMAAwCwYDVR0PBAQDAgXgMG4GA1UdEQRnMGWCE3Rlc3QucGdiYWNrcmVzdC5v cmeCFSoudGVzdC5wZ2JhY2tyZXN0Lm9yZ4IWKi50ZXN0Mi5wZ2JhY2tyZXN0Lm9y -Z4IJMTI3LjAuMC4xhwR/AAABggpkYi1wcmltYXJ5ggpkYi1zdGFuZGJ5ggZiYWNr -dXAwDQYJKoZIhvcNAQELBQADggIBAMbSq1/hjvQZJ2PFE/VVz9OcA7vlewq632eE -P5BalSJJgLVEsv1AxPx8VT08xfFQHQtEcCg/PFqT3RQ5yb1kHfa6glJkjYIdKQbn -lv9OVc/iutQwKPwk32QQjSgQFb/m0tXv9SlQ+gNTdkK4UKffXPj5rpgwaSiVwuLF -d+3TUpJihS48LLRC27kcL5Ur69/fu0ZD7xZSoCr/n8MUq4f9LwOhBqq+h64wM9cV -V79iPWmEJXoNAJrPYmK+XNhcro071c4m+HR4CCNikjxz/GUUf/NGHWT3pL0Ildku -X3dHmsNRVT/wLqi2v2oa6zr9FfVzjDAdCfnvTLOJ6H6dmofzQUFJBSWfhqGNDR8U -oblwirM2sjaOUjnkBS6Cb26yHSClStI+GZvS0KZfSVd2Qbe4YmtQMTNl/hdZGK3z -ZoqV++idVR+A0NQP8xR4VWqQdq0BR5eQOXDA4wtqvivqlIXpbJvqh1kBHPU9cAF+ -g/t3Wa7EomwLazRaV9djLUpon6wGwScKJGzv+vyQSgXN1tQG9tLV4NCFUKDueUUZ -U/j1t64KF9hp5NU2A16zLp6V5GPIJhufXOYa66AFjV8c880eLd5YlkfzgyYwReOx -7vHkiLylbx2tc6aYUqdwjpMwnkxTsn52BBVxDvXToBIRdq/ea/LnZ/yhpnaac/Um -bJOTMee+ +Z4IJMTI3LjAuMC4xhwR/AAABggNwZzGCA3BnMoIEcmVwbzANBgkqhkiG9w0BAQsF +AAOCAgEAX0MEXH6ANllRhdQz6neQ7SVG48Aj1lEAGeOhfpoNKzuyBcRjVw7+NyNN +IwlPKSSBDpnxaWQ5rCLtBtXod6yPMGKTRlFHwFFzfOps6nlRQjPsA848d6daLBvj +unpUQx4NFGPZJSs6z5z4BlT/+5mJVHC9qsrZBtkndYpRWo37xbVhRqP0+FSTbzrx +Gj2td3PoqQBgA/AmSKIpwagGzw7cSor1r4uEjkVxxyOMRbjuuASHMHUM7MtQV3YR +rz9UspvGfoKBdUkzMoqKRwxZWuh+uAoM+1GWXBjqlN6uAdQxpV2wZ75iRJp3Y4Bk +/CkXTLZ83lARGLqS/E5EFfg7Z9Bre2f5fHzV8C/h6WGpvCt/GlZqTx8fix/mMPT2 +CFq+FcSmvF5JsIMxUnpvTw1hcTDNRPnOkFKnO1bjf9+jGCwzDUoGReYpb5veFSxh +IFkQ3oyw3/6v11aPstXSADTvRTFMyRklqu5NIHkMVQCPwQCAE3346KpEsT5HO2T7 +X57sP05alMESjUv1sR260yYC3GUr0dbmf8gthVDhH1SsP3Drn/A3l70xGASnJA46 +LCZwAJZ0KI6G/ok17lTZe9Hjwn2DkmVf1CKD7gXjmroYQI9O+etUtD/g+B2AISTG +SP60NakteOBWqmcSirV5npKh/SZR8Il5oaLS+3HerhgwvNDJ078= -----END CERTIFICATE----- diff --git a/test/ci.pl b/test/ci.pl index 555324300..824402c77 100755 --- a/test/ci.pl +++ b/test/ci.pl @@ -183,7 +183,7 @@ eval # Build list of packages that need to be installed my $strPackage = "make gcc ccache python3-pip git rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config" . - " uncrustify libssh2-1-dev"; + " uncrustify libssh2-1-dev valgrind"; # Add lcov when testing coverage if (vmCoverageC($strVm)) @@ -194,7 +194,7 @@ eval # Extra packages required when testing without containers if ($strVm eq VM_NONE) { - $strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev"; + $strPackage .= " liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev"; } # Else packages needed for integration tests on containers else diff --git a/test/define.yaml b/test/define.yaml index 324648ebb..9334c6316 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -608,7 +608,9 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: sftp total: 19 - harness: libSsh2 + harness: + name: libSsh2 + integration: false harness: name: fd shim: @@ -635,7 +637,9 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: client total: 1 - harness: pq + harness: + name: pq + integration: false coverage: - postgres/client @@ -881,6 +885,7 @@ unit: total: 12 harness: name: backup + integration: false shim: command/backup/backup: function: @@ -1004,13 +1009,14 @@ unit: integration: # ******************************************************************************************************************************** - - name: real + - name: integration db: true test: # ---------------------------------------------------------------------------------------------------------------------------- - name: all total: 1 + harness: host # ********************************************************************************************************************************** # Performance tests diff --git a/test/lib/pgBackRestTest/Common/CoverageTest.pm b/test/lib/pgBackRestTest/Common/CoverageTest.pm index 13c860c7f..4a681aab2 100644 --- a/test/lib/pgBackRestTest/Common/CoverageTest.pm +++ b/test/lib/pgBackRestTest/Common/CoverageTest.pm @@ -26,7 +26,31 @@ use pgBackRestTest::Common::ContainerTest; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::ListTest; -use pgBackRestTest::Common::RunTest; + +#################################################################################################################################### +# testRunName +# +# Create module/test names by upper-casing the first letter and then inserting capitals after each -. +#################################################################################################################################### +sub testRunName +{ + my $strName = shift; + my $bInitCapFirst = shift; + + $bInitCapFirst = defined($bInitCapFirst) ? $bInitCapFirst : true; + my $bFirst = true; + + my @stryName = split('\-', $strName); + $strName = undef; + + foreach my $strPart (@stryName) + { + $strName .= ($bFirst && $bInitCapFirst) || !$bFirst ? ucfirst($strPart) : $strPart; + $bFirst = false; + } + + return $strName; +} #################################################################################################################################### # Generate an lcov configuration file diff --git a/test/lib/pgBackRestTest/Common/FileTest.pm b/test/lib/pgBackRestTest/Common/FileTest.pm deleted file mode 100644 index 6fddeddd2..000000000 --- a/test/lib/pgBackRestTest/Common/FileTest.pm +++ /dev/null @@ -1,277 +0,0 @@ -#################################################################################################################################### -# CommonTest.pm - Common globals used for testing -#################################################################################################################################### -package pgBackRestTest::Common::FileTest; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path cwd); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use File::Copy qw(move); -use File::Path qw(remove_tree); -use IO::Select; -use IPC::Open3; -use POSIX ':sys_wait_h'; -use Symbol 'gensym'; - -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::Host::HostS3Test; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# testLinkCreate -# -# Create a symlink -#################################################################################################################################### -sub testLinkCreate -{ - my $strLink = shift; - my $strDestination = shift; - - # Create the file - symlink($strDestination, $strLink) - or confess "unable to link ${strLink} to ${strDestination}"; -} - -push(@EXPORT, qw(testLinkCreate)); - -#################################################################################################################################### -# testPathMode -# -# Set mode of an existing path. -#################################################################################################################################### -sub testPathMode -{ - my $strPath = shift; - my $strMode = shift; - - # Set the mode - chmod(oct($strMode), $strPath) - or confess 'unable to set mode ${strMode} for ${strPath}'; -} - -push(@EXPORT, qw(testPathMode)); - -#################################################################################################################################### -# testPathRemove -# -# Remove a path and all subpaths. -#################################################################################################################################### -sub testPathRemove -{ - my $strPath = shift; - my $bSuppressError = shift; - - executeTest('rm -rf ' . $strPath, {bSuppressError => $bSuppressError}); -} - -push(@EXPORT, qw(testPathRemove)); - -#################################################################################################################################### -# testFileCreate -# -# Create a file specifying content, mode, and time. -#################################################################################################################################### -sub testFileCreate -{ - my $strFile = shift; - my $strContent = shift; - my $lTime = shift; - my $strMode = shift; - - # Open the file and save strContent to it - my $hFile = shift; - - open($hFile, '>', $strFile) - or confess "unable to open ${strFile} for writing"; - - if (defined($strContent) && $strContent ne '') - { - syswrite($hFile, $strContent) - or confess "unable to write to ${strFile}: $!"; - } - - close($hFile); - - # Set the time - if (defined($lTime)) - { - utime($lTime, $lTime, $strFile) - or confess 'unable to set time ${lTime} for ${strPath}'; - } - - # Set the mode - chmod(oct(defined($strMode) ? $strMode : '0600'), $strFile) - or confess 'unable to set mode ${strMode} for ${strFile}'; -} - -push(@EXPORT, qw(testFileCreate)); - -#################################################################################################################################### -# testFileRemove -# -# Remove a file. -#################################################################################################################################### -sub testFileRemove -{ - my $strFile = shift; - - unlink($strFile) - or confess "unable to remove ${strFile}: $!"; -} - -push(@EXPORT, qw(testFileRemove)); - -#################################################################################################################################### -# forceStorageMode - force mode on a file or path -#################################################################################################################################### -sub forceStorageMode -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strPath, - $strMode, - $bRecurse - ) = - logDebugParam - ( - __PACKAGE__ . '::forceStorageMode', \@_, - {name => 'oStorage'}, - {name => 'strPath'}, - {name => 'strMode'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - # Mode commands are ignored on object storage - if ($oStorage->type() ne STORAGE_OBJECT) - { - executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} ${strPath}"); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageMode)); - -#################################################################################################################################### -# forceStorageMove - force move a directory or file -#################################################################################################################################### -sub forceStorageMove -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strSourcePath, - $strDestinationPath, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->forceStorageMove', \@_, - {name => 'oStorage'}, - {name => 'strSourcePath'}, - {name => 'strDestinationPath'}, - {name => 'bRecurse', optional => true, default => true}, - ); - - # If object storage then use storage commands to remove - if ($oStorage->type() eq STORAGE_OBJECT) - { - if ($bRecurse) - { - my $rhManifest = $oStorage->manifest($strSourcePath); - - foreach my $strName (sort(keys(%{$rhManifest}))) - { - if ($rhManifest->{$strName}{type} eq 'f') - { - $oStorage->put( - "${strDestinationPath}/${strName}", ${$oStorage->get("${strSourcePath}/${strName}", {bRaw => true})}, - {bRaw => true}); - } - } - - $oStorage->pathRemove($strSourcePath, {bRecurse => true}); - } - else - { - $oStorage->put($strDestinationPath, ${$oStorage->get($strSourcePath, {bRaw => true})}, {bRaw => true}); - $oStorage->remove($strSourcePath); - } - } - # Else remove using filesystem commands - else - { - executeTest("mv ${strSourcePath} ${strDestinationPath}"); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageMove)); - -#################################################################################################################################### -# forceStorageRemove - force remove a file or path from storage -#################################################################################################################################### -sub forceStorageRemove -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strPath, - $bRecurse - ) = - logDebugParam - ( - __PACKAGE__ . '->forceStorageRemove', \@_, - {name => 'oStorage'}, - {name => 'strPath'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - # If object storage then use storage commands to remove - if ($oStorage->type() eq STORAGE_OBJECT) - { - $oStorage->pathRemove($strPath, {bRecurse => true}); - } - else - { - executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $strPath); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageRemove)); - -1; diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 0a02b5de5..55fd291d3 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -30,7 +30,6 @@ use pgBackRestTest::Common::DbVersion; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::ListTest; -use pgBackRestTest::Common::RunTest; use pgBackRestTest::Common::VmTest; #################################################################################################################################### @@ -115,7 +114,9 @@ sub new $self->{iTry} = 0; # Setup the path where unit test will be built - $self->{strUnitPath} = "$self->{strTestPath}/unit-$self->{iVmIdx}/$self->{oTest}->{&TEST_VM}"; + $self->{strUnitPath} = + "$self->{strTestPath}/unit-$self->{iVmIdx}/" . + ($self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? 'none' : $self->{oTest}->{&TEST_VM}); $self->{strDataPath} = "$self->{strTestPath}/data-$self->{iVmIdx}"; $self->{strRepoPath} = "$self->{strTestPath}/repo"; @@ -246,58 +247,32 @@ sub run if (!$self->{bDryRun} || $self->{bVmOut}) { - my $strCommand = undef; # Command to run test + my $bValgrind = $self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE; + my $strValgrindSuppress = + $self->{strRepoPath} . '/test/src/valgrind.suppress.' . + ($self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? VM_NONE : $self->{oTest}->{&TEST_VM}); + my $strVm = $self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? VM_NONE : $self->{oTest}->{&TEST_VM}; - # If testing with C harness - if ($self->{oTest}->{&TEST_C}) - { - # Create command - # ------------------------------------------------------------------------------------------------------------------ - # Build filename for valgrind suppressions - my $strValgrindSuppress = $self->{strRepoPath} . '/test/src/valgrind.suppress.' . $self->{oTest}->{&TEST_VM}; - - $strCommand = - ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '" : '') . - " \\\n" . - $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM} . '/test/src/test-pgbackrest' . - ' --repo-path=' . $self->{strTestPath} . '/repo' . ' --test-path=' . $self->{strTestPath} . - " --log-level=$self->{strLogLevel}" . ' --vm=' . $self->{oTest}->{&TEST_VM} . - ' --vm-id=' . $self->{iVmIdx} . ($self->{bProfile} ? ' --profile' : '') . - ($self->{bLogTimestamp} ? '' : ' --no-log-timestamp') . - ($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') . - ($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . ' test ' . - $self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" . - # Allow stderr to be copied to stderr and stdout - "exec 3>&1 && \\\n" . - # Test with valgrind when requested - ($self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE ? - 'valgrind -q --gen-suppressions=all' . - ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') . - " --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') . - "$self->{strUnitPath}/build/test-unit 2>&1 1>&3 | tee /dev/stderr" . - ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "'" : ''); - } - else - { - $strCommand = - ($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . - abs_path($0) . - " --test-path=${strVmTestPath}" . - " --vm=$self->{oTest}->{&TEST_VM}" . - " --vm-id=$self->{iVmIdx}" . - " --module=" . $self->{oTest}->{&TEST_MODULE} . - ' --test=' . $self->{oTest}->{&TEST_NAME} . - $strCommandRunParam . - (defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') . - ($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') . - ($self->{strLogLevelTestFile} ne lc(TRACE) ? " --log-level-test-file=$self->{strLogLevelTestFile}" : '') . + my $strCommand = + ($strVm ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '\\\n" : '') . + $self->{strTestPath} . "/build/${strVm}/test/src/test-pgbackrest" . + ' --repo-path=' . $self->{strTestPath} . '/repo' . ' --test-path=' . $self->{strTestPath} . + " --log-level=$self->{strLogLevel}" . ' --vm=' . $self->{oTest}->{&TEST_VM} . + ' --vm-id=' . $self->{iVmIdx} . ($self->{bProfile} ? ' --profile' : '') . ($self->{bLogTimestamp} ? '' : ' --no-log-timestamp') . - ' --psql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} . ($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') . - ($self->{bDryRun} ? ' --dry-run' : '') . - ($self->{bDryRun} ? ' --vm-out' : '') . - ($self->{bNoCleanup} ? " --no-cleanup" : ''); - } + (defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') . + ($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . ' test ' . + $self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" . + # Allow stderr to be copied to stderr and stdout + "exec 3>&1 && \\\n" . + # Test with valgrind when requested + ($bValgrind ? + 'valgrind -q --gen-suppressions=all' . + ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') . + " --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') . + "$self->{strUnitPath}/build/test-unit 2>&1 1>&3 | tee /dev/stderr" . + ($strVm ne VM_NONE ? "'" : ''); my $oExec = new pgBackRestTest::Common::ExecuteTest( $strCommand, {bSuppressError => true, bShowOutputAsync => $self->{bShowOutputAsync}}); diff --git a/test/lib/pgBackRestTest/Common/RunTest.pm b/test/lib/pgBackRestTest/Common/RunTest.pm deleted file mode 100644 index 2c1ce7e4e..000000000 --- a/test/lib/pgBackRestTest/Common/RunTest.pm +++ /dev/null @@ -1,426 +0,0 @@ -#################################################################################################################################### -# RunTest.pm - All tests are inherited from this object -#################################################################################################################################### -package pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::BuildTest; -use pgBackRestTest::Common::DefineTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::Storage; -use pgBackRestTest::Common::StoragePosix; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# Constant to use when bogus data is required -#################################################################################################################################### -use constant BOGUS => 'bogus'; - push @EXPORT, qw(BOGUS); - -#################################################################################################################################### -# The current test run that is executing. Only a single run should ever occur in a process to prevent various cleanup issues from -# affecting the next run. Of course multiple subtests can be executed in a single run. -#################################################################################################################################### -my $oTestRun; -my $oStorage; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->new'); - - # Initialize run counter - $self->{iRun} = 0; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# initModule -# -# Empty init sub in case the ancestor class does not declare one. -#################################################################################################################################### -sub initModule {} - -#################################################################################################################################### -# initTest -# -# Empty init sub in case the ancestor class does not declare one. -#################################################################################################################################### -sub initTest {} - -#################################################################################################################################### -# cleanTest -# -# Delete all files in test directory. -#################################################################################################################################### -sub cleanTest -{ - my $self = shift; - - executeTest('rm -rf ' . $self->testPath() . '/*'); -} - -#################################################################################################################################### -# cleanModule -# -# Empty final sub in case the ancestor class does not declare one. -#################################################################################################################################### -sub cleanModule {} - -#################################################################################################################################### -# process -#################################################################################################################################### -sub process -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{strVm}, - $self->{iVmId}, - $self->{strBasePath}, - $self->{strTestPath}, - $self->{strBackRestExe}, - $self->{strBackRestExeHelper}, - $self->{strPgBinPath}, - $self->{strPgVersion}, - $self->{strModule}, - $self->{strModuleTest}, - $self->{iyModuleTestRun}, - $self->{bOutput}, - $self->{bDryRun}, - $self->{bCleanup}, - $self->{strLogLevelTestFile}, - $self->{strPgUser}, - $self->{strGroup}, - ) = - logDebugParam - ( - __PACKAGE__ . '->process', \@_, - {name => 'strVm'}, - {name => 'iVmId'}, - {name => 'strBasePath'}, - {name => 'strTestPath'}, - {name => 'strBackRestExe'}, - {name => 'strBackRestExeHelper'}, - {name => 'strPgBinPath', required => false}, - {name => 'strPgVersion', required => false}, - {name => 'strModule'}, - {name => 'strModuleTest'}, - {name => 'iModuleTestRun', required => false}, - {name => 'bOutput'}, - {name => 'bDryRun'}, - {name => 'bCleanup'}, - {name => 'strLogLevelTestFile'}, - {name => 'strPgUser'}, - {name => 'strGroup'}, - ); - - # Init will only be run on first test, clean/init on subsequent tests - $self->{bFirstTest} = true; - - # Initialize test storage - $oStorage = new pgBackRestTest::Common::Storage( - $self->testPath(), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); - - # Init, run, and clean the test(s) - $self->initModule(); - $self->run(); - $self->cleanModule(); - - # Make sure the correct number of tests ran - my $hModuleTest = testDefModuleTest($self->{strModule}, $self->{strModuleTest}); - - if ($hModuleTest->{&TESTDEF_TOTAL} != $self->runCurrent()) - { - confess &log(ASSERT, "expected $hModuleTest->{&TESTDEF_TOTAL} tests to run but $self->{iRun} ran"); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# begin -#################################################################################################################################### -sub begin -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDescription, - ) = - logDebugParam - ( - __PACKAGE__ . '->begin', \@_, - {name => 'strDescription'}, - ); - - # Increment the run counter; - $self->{iRun}++; - - # Return if this test should not be run - if (@{$self->{iyModuleTestRun}} != 0 && !grep(/^$self->{iRun}$/i, @{$self->{iyModuleTestRun}})) - { - return false; - } - - # Output information about test to run - &log(INFO, 'run ' . sprintf('%03d', $self->runCurrent()) . ' - ' . $strDescription); - - if ($self->isDryRun()) - { - return false; - } - - if (!$self->{bFirstTest}) - { - $self->cleanTest(); - } - - $self->initTest(); - $self->{bFirstTest} = false; - - return true; -} - -#################################################################################################################################### -# testResult -#################################################################################################################################### -sub testResult -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $fnSub, - $strExpected, - $strDescription, - $iWaitSeconds, - ) = - logDebugParam - ( - __PACKAGE__ . '::testResult', \@_, - {name => 'fnSub', trace => true}, - {name => 'strExpected', required => false, trace => true}, - {name => 'strDescription', trace => true}, - {name => 'iWaitSeconds', optional => true, default => 0, trace => true}, - ); - - &log(INFO, ' ' . $strDescription); - my $strActual; - my $bWarnValid = true; - - my $oWait = waitInit($iWaitSeconds); - my $bDone = false; - - # Clear the cache for this test - logFileCacheClear(); - - my @stryResult; - - do - { - eval - { - @stryResult = ref($fnSub) eq 'CODE' ? $fnSub->() : $fnSub; - - if (@stryResult <= 1) - { - $strActual = ${logDebugBuild($stryResult[0])}; - } - else - { - $strActual = ${logDebugBuild(\@stryResult)}; - } - - return true; - } - or do - { - if (!isException(\$EVAL_ERROR)) - { - confess "unexpected standard Perl exception" . (defined($EVAL_ERROR) ? ": ${EVAL_ERROR}" : ''); - } - - confess &logException($EVAL_ERROR); - }; - - if ($strActual ne (defined($strExpected) ? $strExpected : "[undef]")) - { - if (!waitMore($oWait)) - { - confess - "expected:\n" . (defined($strExpected) ? "\"${strExpected}\"" : '[undef]') . - "\nbut actual was:\n" . (defined($strActual) ? "\"${strActual}\"" : '[undef]'); - } - } - else - { - $bDone = true; - } - } while (!$bDone); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'result', value => \@stryResult, trace => true} - ); -} - -#################################################################################################################################### -# testRunName -# -# Create module/test names by upper-casing the first letter and then inserting capitals after each -. -#################################################################################################################################### -sub testRunName -{ - my $strName = shift; - my $bInitCapFirst = shift; - - $bInitCapFirst = defined($bInitCapFirst) ? $bInitCapFirst : true; - my $bFirst = true; - - my @stryName = split('\-', $strName); - $strName = undef; - - foreach my $strPart (@stryName) - { - $strName .= ($bFirst && $bInitCapFirst) || !$bFirst ? ucfirst($strPart) : $strPart; - $bFirst = false; - } - - return $strName; -} - -push @EXPORT, qw(testRunName); - -#################################################################################################################################### -# testRun -#################################################################################################################################### -sub testRun -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strModule, - $strModuleTest, - ) = - logDebugParam - ( - __PACKAGE__ . '::testRun', \@_, - {name => 'strModule', trace => true}, - {name => 'strModuleTest', trace => true}, - ); - - # Error if the test run is already defined - only one run per process is allowed - if (defined($oTestRun)) - { - confess &log(ASSERT, 'a test run has already been created in this process'); - } - - my $strModuleName = - 'pgBackRestTest::Module::' . testRunName($strModule) . '::' . testRunName($strModule) . testRunName($strModuleTest) . - 'Test'; - - $oTestRun = eval("require ${strModuleName}; ${strModuleName}->import(); return new ${strModuleName}();") - or do {confess $EVAL_ERROR}; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oRun', value => $oTestRun, trace => true} - ); -} - -push @EXPORT, qw(testRun); - -#################################################################################################################################### -# testRunGet -#################################################################################################################################### -sub testRunGet -{ - return $oTestRun; -} - -push @EXPORT, qw(testRunGet); - -#################################################################################################################################### -# storageTest - get the storage for the current test -#################################################################################################################################### -sub storageTest -{ - return $oStorage; -} - -push(@EXPORT, qw(storageTest)); - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub archBits {return vmArchBits(shift->{strVm})} -sub backrestExe {return shift->{strBackRestExe}} -sub backrestExeHelper {return shift->{strBackRestExeHelper}} -sub basePath {return shift->{strBasePath}} -sub dataPath {return shift->basePath() . '/test/data'} -sub doCleanup {return shift->{bCleanup}} -sub logLevelTestFile {return shift->{strLogLevelTestFile}} -sub group {return shift->{strGroup}} -sub isDryRun {return shift->{bDryRun}} -sub module {return shift->{strModule}} -sub moduleTest {return shift->{strModuleTest}} -sub pgBinPath {return shift->{strPgBinPath}} -sub pgUser {return shift->{strPgUser}} -sub pgVersion {return shift->{strPgVersion}} -sub runCurrent {return shift->{iRun}} -sub stanza {return 'db'} -sub testPath {return shift->{strTestPath}} -sub vm {return shift->{strVm}} -sub vmId {return shift->{iVmId}} - -1; diff --git a/test/lib/pgBackRestTest/Common/StorageRepo.pm b/test/lib/pgBackRestTest/Common/StorageRepo.pm deleted file mode 100644 index 7469342d2..000000000 --- a/test/lib/pgBackRestTest/Common/StorageRepo.pm +++ /dev/null @@ -1,623 +0,0 @@ -#################################################################################################################################### -# C Storage Interface -#################################################################################################################################### -package pgBackRestTest::Common::StorageRepo; -use parent 'pgBackRestTest::Common::StorageBase'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Digest::SHA qw(sha1_hex); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Fcntl qw(:mode); -use File::stat qw{lstat}; -use JSON::PP; - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::Io::Handle; -use pgBackRestTest::Common::Io::Process; -use pgBackRestTest::Common::StorageBase; - -#################################################################################################################################### -# Temp file extension -#################################################################################################################################### -use constant STORAGE_TEMP_EXT => PROJECT_EXE . '.tmp'; - push @EXPORT, qw(STORAGE_TEMP_EXT); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{strCommand}, - $self->{strType}, - $self->{lBufferMax}, - $self->{iTimeoutIo}, - $self->{iRepo}, - $self->{strDefaultPathMode}, - $self->{strDefaultFileMode}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strCommand'}, - {name => 'strType'}, - {name => 'lBufferMax'}, - {name => 'iTimeoutIo'}, - {name => 'iRepo'}, - {name => 'strDefaultPathMode', optional => true, default => '0750'}, - {name => 'strDefaultFileMode', optional => true, default => '0640'}, - ); - - # Create JSON object - $self->{oJSON} = JSON::PP->new()->allow_nonref(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# Escape characteres that have special meaning on the command line -#################################################################################################################################### -sub escape -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strValue, - ) = - logDebugParam - ( - __PACKAGE__ . '->escape', \@_, - {name => 'strValue', trace => true}, - ); - - $strValue =~ s/\\/\\\\/g; - $strValue =~ s/\/\\\>/g; - $strValue =~ s/\!/\\\!/g; - $strValue =~ s/\*/\\\*/g; - $strValue =~ s/\(/\\\(/g; - $strValue =~ s/\)/\\\)/g; - $strValue =~ s/\&/\\\&/g; - $strValue =~ s/\'/\\\'/g; - $strValue =~ s/\;/\\\;/g; - $strValue =~ s/\?/\\\?/g; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strValue', value => $strValue}, - ); -} - -#################################################################################################################################### -# Execute command and return the output -#################################################################################################################################### -sub exec -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strCommand, - ) = - logDebugParam - ( - __PACKAGE__ . '->exec', \@_, - {name => 'strCommand'}, - ); - - $strCommand = "$self->{strCommand} ${strCommand}"; - my $oBuffer = new pgBackRestTest::Common::Io::Buffered( - new pgBackRestTest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax}); - my $oProcess = new pgBackRestTest::Common::Io::Process($oBuffer, $strCommand); - - my $tResult; - - while (!$oBuffer->eof()) - { - $oBuffer->read(\$tResult, $self->{lBufferMax}, false); - } - - $oProcess->close(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'tResult', value => $tResult}, - {name => 'iExitStatus', value => $oProcess->exitStatus()}, - ); -} - -#################################################################################################################################### -# Create storage -#################################################################################################################################### -sub create -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->create'); - - $self->exec("--repo=$self->{iRepo} repo-create"); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Check if file exists (not a path) -#################################################################################################################################### -sub exists -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileExp, - ) = - logDebugParam - ( - __PACKAGE__ . '->exists', \@_, - {name => 'strFileExp'}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bExists', value => $self->info($strFileExp, {bIgnoreMissing => true})->{type} eq 'f'} - ); -} - -#################################################################################################################################### -# Read a buffer from storage all at once -#################################################################################################################################### -sub get -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $xFile, - $strCipherPass, - $bRaw, - ) = - logDebugParam - ( - __PACKAGE__ . '->get', \@_, - {name => 'xFile', required => false}, - {name => 'strCipherPass', optional => true, redact => true}, - {name => 'bRaw', optional => true, default => false}, - ); - - # If openRead() was called first set values from that call - my $strFile = $xFile; - my $bIgnoreMissing = false; - - if (ref($xFile)) - { - $strFile = $xFile->{strFile}; - $bIgnoreMissing = $xFile->{bIgnoreMissing}; - $strCipherPass = $xFile->{strCipherPass}; - } - - # Check invalid params - if ($bRaw && defined($strCipherPass)) - { - confess &log(ERROR, 'bRaw and strCipherPass cannot both be set'); - } - - # Get file - my ($tResult, $iExitStatus) = $self->exec( - (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . ($bRaw ? ' --raw' : '') . - ($bIgnoreMissing ? ' --ignore-missing' : '') . " --repo=$self->{iRepo} repo-get " . $self->escape($strFile)); - - # Error if missing an not ignored - if ($iExitStatus == 1 && !$bIgnoreMissing) - { - confess &log(ERROR, "unable to open '${strFile}'", ERROR_FILE_OPEN); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rtContent', value => $iExitStatus == 0 ? \$tResult : undef, trace => true}, - ); -} - -#################################################################################################################################### -# Get information for path/file -#################################################################################################################################### -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathFileExp, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strPathFileExp'}, - {name => 'bIgnoreMissing', optional => true, default => false}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rhInfo', value => $self->manifest($strPathFileExp, {bRecurse => false})->{'.'}, trace => true} - ); -} - -#################################################################################################################################### -# List all files/paths in path -#################################################################################################################################### -sub list -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathExp, - $strExpression, - $strSortOrder, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->list', \@_, - {name => 'strPathExp', required => false}, - {name => 'strExpression', optional => true}, - {name => 'strSortOrder', optional => true, default => 'forward'}, - {name => 'bIgnoreMissing', optional => true, default => false}, - ); - - # Get file list - my $rstryFileList = []; - my $rhManifest = $self->manifest($strPathExp, {bRecurse => false}); - - foreach my $strKey ($strSortOrder eq 'reverse' ? sort {$b cmp $a} keys(%{$rhManifest}) : sort keys(%{$rhManifest})) - { - next if $strKey eq '.'; - next if defined($strExpression) && $strKey !~ $strExpression; - - push(@{$rstryFileList}, $strKey); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryFileList', value => $rstryFileList} - ); -} - -#################################################################################################################################### -# Build path/file/link manifest starting with base path and including all subpaths -#################################################################################################################################### -sub manifest -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathExp, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifest', \@_, - {name => 'strPathExp'}, - {name => 'bRecurse', optional => true, default => true}, - ); - - my $rhManifest = $self->{oJSON}->decode( - $self->exec( - "--output=json" . ($bRecurse ? ' --recurse' : '') . " --repo=$self->{iRepo} repo-ls " . $self->escape($strPathExp))); - - # Transform the manifest to the old format - foreach my $strKey (keys(%{$rhManifest})) - { - if ($rhManifest->{$strKey}{type} eq 'file') - { - $rhManifest->{$strKey}{type} = 'f'; - - if (defined($rhManifest->{$strKey}{time})) - { - $rhManifest->{$strKey}{modified_time} = $rhManifest->{$strKey}{time}; - delete($rhManifest->{$strKey}{time}); - } - } - elsif ($rhManifest->{$strKey}{type} eq 'path') - { - $rhManifest->{$strKey}{type} = 'd'; - } - elsif ($rhManifest->{$strKey}{type} eq 'link') - { - $rhManifest->{$strKey}{type} = 'l'; - } - elsif ($rhManifest->{$strKey}{type} eq 'special') - { - $rhManifest->{$strKey}{type} = 's'; - } - else - { - confess "invalid file type '$rhManifest->{type}'"; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rhManifest', value => $rhManifest, trace => true} - ); -} - -#################################################################################################################################### -# Open file for reading -#################################################################################################################################### -sub openRead -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - $bIgnoreMissing, - $strCipherPass, - ) = - logDebugParam - ( - __PACKAGE__ . '->openRead', \@_, - {name => 'strFile'}, - {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'strCipherPass', optional => true, redact => true}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rhFileIo', value => {strFile => $strFile, bIgnoreMissing => $bIgnoreMissing, strCipherPass => $strCipherPass}, - trace => true}, - ); -} - -#################################################################################################################################### -# Remove path and all files below it -#################################################################################################################################### -sub pathRemove -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathRemove', \@_, - {name => 'strPath'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - $self->exec("--repo=$self->{iRepo} repo-rm " . ($bRecurse ? '--recurse ' : '') . $self->escape($strPath)); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# put - writes a buffer out to storage all at once -#################################################################################################################################### -sub put -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - $tContent, - $strCipherPass, - $bRaw, - ) = - logDebugParam - ( - __PACKAGE__ . '->put', \@_, - {name => 'strFile'}, - {name => 'tContent', required => false}, - {name => 'strCipherPass', optional => true, redact => true}, - {name => 'bRaw', optional => true, default => false}, - ); - - # Check invalid params - if ($bRaw && defined($strCipherPass)) - { - confess &log(ERROR, 'bRaw and strCipherPass cannot both be set'); - } - - # Put file - my $strCommand = - "$self->{strCommand}" . (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . - ($bRaw ? ' --raw' : '') . " --repo=$self->{iRepo} repo-put " . $self->escape($strFile); - - my $oBuffer = new pgBackRestTest::Common::Io::Buffered( - new pgBackRestTest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax}); - my $oProcess = new pgBackRestTest::Common::Io::Process($oBuffer, $strCommand); - - if (defined($tContent)) - { - $oBuffer->write(\$tContent); - } - - close($oBuffer->handleWrite()); - - my $tResult; - - while (!$oBuffer->eof()) - { - $oBuffer->read(\$tResult, $self->{lBufferMax}, false); - } - - close($oBuffer->handleRead()); - $oProcess->close(); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Remove file -#################################################################################################################################### -sub remove -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - ) = - logDebugParam - ( - __PACKAGE__ . '->remove', \@_, - {name => 'xFileExp'}, - ); - - $self->exec("--repo=$self->{iRepo} repo-rm " . $self->escape($strFile)); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Cache storage so it can be retrieved quickly -#################################################################################################################################### -my $oRepoStorage; - -#################################################################################################################################### -# storageRepoCommandSet -#################################################################################################################################### -my $strStorageRepoCommand; -my $strStorageRepoType; - -sub storageRepoCommandSet -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strCommand, - $strStorageType, - ) = - logDebugParam - ( - __PACKAGE__ . '::storageRepoCommandSet', \@_, - {name => 'strCommand'}, - {name => 'strStorageType'}, - ); - - $strStorageRepoCommand = $strCommand; - $strStorageRepoType = $strStorageType; - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push @EXPORT, qw(storageRepoCommandSet); - -#################################################################################################################################### -# storageRepo - get repository storage -#################################################################################################################################### -sub storageRepo -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $iRepo, - ) = - logDebugParam - ( - __PACKAGE__ . '::storageRepo', \@_, - {name => 'strStanza', optional => true, trace => true}, - {name => 'iRepo', optional => true, default => 1, trace => true}, - ); - - # Create storage if not defined - if (!defined($oRepoStorage->{$iRepo})) - { - $oRepoStorage->{$iRepo} = new pgBackRestTest::Common::StorageRepo( - $strStorageRepoCommand, $strStorageRepoType, 64 * 1024, 60, $iRepo); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oStorageRepo', value => $oRepoStorage->{$iRepo}, trace => true}, - ); -} - -push @EXPORT, qw(storageRepo); - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub capability {shift->type() eq STORAGE_POSIX} -sub type {shift->{strType}} - -1; diff --git a/test/lib/pgBackRestTest/Env/ArchiveInfo.pm b/test/lib/pgBackRestTest/Env/ArchiveInfo.pm deleted file mode 100644 index 42257b9ea..000000000 --- a/test/lib/pgBackRestTest/Env/ArchiveInfo.pm +++ /dev/null @@ -1,478 +0,0 @@ -#################################################################################################################################### -# ARCHIVE INFO MODULE -# -# The archive.info file is created when archiving begins. It is located under the stanza directory. The file contains information -# regarding the stanza database version, database WAL segment system id and other information to ensure that archiving is being -# performed on the proper database. -#################################################################################################################################### -package pgBackRestTest::Env::ArchiveInfo; -use parent 'pgBackRestDoc::Common::Ini'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname basename); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# File/path constants -#################################################################################################################################### -use constant ARCHIVE_INFO_FILE => 'archive.info'; - push @EXPORT, qw(ARCHIVE_INFO_FILE); - -#################################################################################################################################### -# RegEx constants -#################################################################################################################################### -use constant REGEX_ARCHIVE_DIR_DB_VERSION => '^[0-9]+(\.[0-9]+)*-[0-9]+$'; - push @EXPORT, qw(REGEX_ARCHIVE_DIR_DB_VERSION); -use constant REGEX_ARCHIVE_DIR_WAL => '^[0-F]{16}$'; - push @EXPORT, qw(REGEX_ARCHIVE_DIR_WAL); - -#################################################################################################################################### -# WAL segment size -#################################################################################################################################### -use constant PG_WAL_SEGMENT_SIZE => 16777216; - push @EXPORT, qw(PG_WAL_SEGMENT_SIZE); - -#################################################################################################################################### -# Archive info constants -#################################################################################################################################### -use constant INFO_ARCHIVE_SECTION_DB => INFO_BACKUP_SECTION_DB; - push @EXPORT, qw(INFO_ARCHIVE_SECTION_DB); -use constant INFO_ARCHIVE_SECTION_DB_HISTORY => INFO_BACKUP_SECTION_DB_HISTORY; - push @EXPORT, qw(INFO_ARCHIVE_SECTION_DB_HISTORY); - -use constant INFO_ARCHIVE_KEY_DB_VERSION => MANIFEST_KEY_DB_VERSION; - push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_VERSION); -use constant INFO_ARCHIVE_KEY_DB_ID => MANIFEST_KEY_DB_ID; - push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_ID); -use constant INFO_ARCHIVE_KEY_DB_SYSTEM_ID => MANIFEST_KEY_SYSTEM_ID; - push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_SYSTEM_ID); - -#################################################################################################################################### -# Global variables -#################################################################################################################################### -my $strArchiveInfoMissingMsg = - ARCHIVE_INFO_FILE . " does not exist but is required to push/get WAL segments\n" . - "HINT: is archive_command configured in postgresql.conf?\n" . - "HINT: has a stanza-create been performed?\n" . - "HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme."; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchiveClusterPath, # Archive cluster path - $bRequired, # Is archive info required? - $bLoad, # Should the file attempt to be loaded? - $bIgnoreMissing, # Don't error on missing files - $strCipherPassSub, # Passphrase to encrypt the subsequent archive files if repo is encrypted - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strArchiveClusterPath'}, - {name => 'bRequired', default => true}, - {name => 'bLoad', optional => true, default => true}, - {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'strCipherPassSub', optional => true}, - ); - - # Build the archive info path/file name - my $strArchiveInfoFile = "${strArchiveClusterPath}/" . ARCHIVE_INFO_FILE; - my $self = {}; - my $iResult = 0; - my $strResultMessage; - - # Init object and store variables - eval - { - $self = $class->SUPER::new( - storageRepo(), $strArchiveInfoFile, - {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub}); - return true; - } - or do - { - # Capture error information - $iResult = exceptionCode($EVAL_ERROR); - $strResultMessage = exceptionMessage($EVAL_ERROR); - }; - - if ($iResult != 0) - { - # If the file does not exist but is required to exist, then error - # The archive info is only allowed not to exist when running a stanza-create on a new install - if ($iResult == ERROR_FILE_MISSING) - { - if ($bRequired) - { - confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING); - } - } - elsif ($iResult == ERROR_CRYPTO && $strResultMessage =~ "^unable to flush") - { - confess &log(ERROR, "unable to parse '$strArchiveInfoFile'\nHINT: is or was the repo encrypted?", $iResult); - } - else - { - confess $EVAL_ERROR; - } - } - - $self->{strArchiveClusterPath} = $strArchiveClusterPath; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# check -# -# Check archive info file and make sure it is compatible with the current version of the database for the stanza. If the file does -# not exist an error will occur. -#################################################################################################################################### -sub check -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $bRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - {name => 'bRequired', default => true}, - ); - - # ??? remove bRequired after stanza-upgrade - if ($bRequired) - { - # Confirm the info file exists with the DB section - $self->confirmExists(); - } - - my $strError = undef; - - if (!$self->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, $strDbVersion)) - { - $strError = "WAL segment version ${strDbVersion} does not match archive version " . - $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION); - } - - if (!$self->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, $ullDbSysId)) - { - $strError = (defined($strError) ? ($strError . "\n") : "") . - "WAL segment system-id ${ullDbSysId} does not match archive system-id " . - $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID); - } - - if (defined($strError)) - { - confess &log(ERROR, "${strError}\nHINT: are you archiving to the correct stanza?", ERROR_ARCHIVE_MISMATCH); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveId', value => $self->archiveId()} - ); -} - -#################################################################################################################################### -# archiveId -# -# Get the archive id which is a combination of the DB version and the db-id setting (e.g. 9.4-1) -#################################################################################################################################### -sub archiveId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - ) = logDebugParam - ( - __PACKAGE__ . '->archiveId', \@_, - {name => 'strDbVersion', optional => true}, - {name => 'ullDbSysId', optional => true}, - ); - - my $strArchiveId = undef; - - # If neither optional version and system-id are passed then set the archive id to the current one - if (!defined($strDbVersion) && !defined($ullDbSysId)) - { - $strArchiveId = $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION) . "-" . - $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID); - } - # If both the optional version and system-id are passed - elsif (defined($strDbVersion) && defined($ullDbSysId)) - { - # Get the newest archiveId for the version/system-id passed - $strArchiveId = ($self->archiveIdList($strDbVersion, $ullDbSysId))[0]; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveId', value => $strArchiveId} - ); -} - -#################################################################################################################################### -# archiveIdList -# -# Get a sorted list of the archive ids for the db-version and db-system-id passed. -#################################################################################################################################### -sub archiveIdList -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - ) = logDebugParam - ( - __PACKAGE__ . '->archiveIdList', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - ); - - my @stryArchiveId; - - # Get the version and system-id for all known databases - my $hDbList = $self->dbHistoryList(); - - foreach my $iDbHistoryId (sort {$a <=> $b} keys %$hDbList) - { - # If the version and system-id match then construct the archive id so that the constructed array has the newest match first - if (($hDbList->{$iDbHistoryId}{&INFO_DB_VERSION} eq $strDbVersion) && - ($hDbList->{$iDbHistoryId}{&INFO_SYSTEM_ID} eq $ullDbSysId)) - { - unshift(@stryArchiveId, $strDbVersion . "-" . $iDbHistoryId); - } - } - - # If the archive id has still not been found, then error - if (@stryArchiveId == 0) - { - confess &log( - ERROR, "unable to retrieve the archive id for database version '$strDbVersion' and system-id '$ullDbSysId'", - ERROR_ARCHIVE_MISMATCH); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryArchiveId', value => \@stryArchiveId} - ); -} - -#################################################################################################################################### -# create -# -# Creates the archive.info file. WARNING - this function should only be called from stanza-create or tests. -#################################################################################################################################### -sub create -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->create', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - {name => 'bSave', default => true}, - ); - - # Fill db section and db history section - $self->dbSectionSet($strDbVersion, $ullDbSysId, $self->dbHistoryIdGet(false)); - - if ($bSave) - { - $self->save(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# dbHistoryIdGet -# -# Get the db history ID -#################################################################################################################################### -sub dbHistoryIdGet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $bFileRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbHistoryIdGet', \@_, - {name => 'bFileRequired', default => true}, - ); - - # Confirm the info file exists if it is required - if ($bFileRequired) - { - $self->confirmExists(); - } - - # If the DB section does not exist, initialize the history to one, else return the latest ID - my $iDbHistoryId = (!$self->test(INFO_ARCHIVE_SECTION_DB)) - ? 1 : $self->numericGet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $iDbHistoryId} - ); -} - -#################################################################################################################################### -# dbHistoryList -# -# Get the data from the db history section. -#################################################################################################################################### -sub dbHistoryList -{ - my $self = shift; - my - ( - $strOperation, - ) = logDebugParam - ( - __PACKAGE__ . '->dbHistoryList', - ); - - my %hDbHash; - - foreach my $iHistoryId ($self->keys(INFO_ARCHIVE_SECTION_DB_HISTORY)) - { - $hDbHash{$iHistoryId}{&INFO_DB_VERSION} = - $self->get(INFO_ARCHIVE_SECTION_DB_HISTORY, $iHistoryId, INFO_ARCHIVE_KEY_DB_VERSION); - $hDbHash{$iHistoryId}{&INFO_SYSTEM_ID} = - $self->get(INFO_ARCHIVE_SECTION_DB_HISTORY, $iHistoryId, INFO_ARCHIVE_KEY_DB_ID); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hDbHash', value => \%hDbHash} - ); -} - -#################################################################################################################################### -# dbSectionSet -# -# Set the db and db:history sections. -#################################################################################################################################### -sub dbSectionSet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $iDbHistoryId, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbSectionSet', \@_, - {name => 'strDbVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - {name => 'iDbHistoryId', trace => true} - ); - - # Fill db section - $self->numericSet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, $ullDbSysId); - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, $strDbVersion . ''); - $self->numericSet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID, undef, $iDbHistoryId); - - # Fill db history - $self->numericSet(INFO_ARCHIVE_SECTION_DB_HISTORY, $iDbHistoryId, INFO_ARCHIVE_KEY_DB_ID, $ullDbSysId); - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(INFO_ARCHIVE_SECTION_DB_HISTORY, $iDbHistoryId, INFO_ARCHIVE_KEY_DB_VERSION, $strDbVersion . ''); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# confirmExists -# -# Ensure that the archive.info file and the db section exist. -#################################################################################################################################### -sub confirmExists -{ - my $self = shift; - - # Confirm the file exists and the DB section is filled out - if (!$self->test(INFO_ARCHIVE_SECTION_DB) || !$self->{bExists}) - { - confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Env/BackupInfo.pm b/test/lib/pgBackRestTest/Env/BackupInfo.pm deleted file mode 100644 index 3b8b4061a..000000000 --- a/test/lib/pgBackRestTest/Env/BackupInfo.pm +++ /dev/null @@ -1,898 +0,0 @@ -#################################################################################################################################### -# BACKUP INFO MODULE -#################################################################################################################################### -package pgBackRestTest::Env::BackupInfo; -use parent 'pgBackRestDoc::Common::Ini'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname basename); -use File::stat; - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Backup type constants -#################################################################################################################################### -use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL); -use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF); -use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR); - -#################################################################################################################################### -# File/path constants -#################################################################################################################################### -use constant FILE_BACKUP_INFO => 'backup.info'; - push @EXPORT, qw(FILE_BACKUP_INFO); - -#################################################################################################################################### -# Backup info constants -#################################################################################################################################### -use constant INFO_BACKUP_SECTION_BACKUP => MANIFEST_SECTION_BACKUP; - push @EXPORT, qw(INFO_BACKUP_SECTION_BACKUP); -use constant INFO_BACKUP_SECTION_BACKUP_CURRENT => INFO_BACKUP_SECTION_BACKUP . ':current'; - push @EXPORT, qw(INFO_BACKUP_SECTION_BACKUP_CURRENT); - -use constant INFO_BACKUP_KEY_ARCHIVE_CHECK => MANIFEST_KEY_ARCHIVE_CHECK; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_CHECK); -use constant INFO_BACKUP_KEY_ARCHIVE_COPY => MANIFEST_KEY_ARCHIVE_COPY; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_COPY); -use constant INFO_BACKUP_KEY_ARCHIVE_START => MANIFEST_KEY_ARCHIVE_START; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_START); -use constant INFO_BACKUP_KEY_ARCHIVE_STOP => MANIFEST_KEY_ARCHIVE_STOP; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_STOP); -use constant INFO_BACKUP_KEY_BACKUP_STANDBY => MANIFEST_KEY_BACKUP_STANDBY; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_STANDBY); -use constant INFO_BACKUP_KEY_BACKUP_REPO_SIZE => 'backup-info-repo-size'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_REPO_SIZE); -use constant INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA => 'backup-info-repo-size-delta'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA); -use constant INFO_BACKUP_KEY_BACKUP_SIZE => 'backup-info-size'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_SIZE); -use constant INFO_BACKUP_KEY_BACKUP_SIZE_DELTA => 'backup-info-size-delta'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_SIZE_DELTA); -use constant INFO_BACKUP_KEY_CATALOG => MANIFEST_KEY_CATALOG; - push @EXPORT, qw(INFO_BACKUP_KEY_CATALOG); -use constant INFO_BACKUP_KEY_CONTROL => MANIFEST_KEY_CONTROL; - push @EXPORT, qw(INFO_BACKUP_KEY_CONTROL); -use constant INFO_BACKUP_KEY_COMPRESS => MANIFEST_KEY_COMPRESS; - push @EXPORT, qw(INFO_BACKUP_KEY_COMPRESS); -use constant INFO_BACKUP_KEY_CHECKSUM_PAGE => MANIFEST_KEY_CHECKSUM_PAGE; - push @EXPORT, qw(INFO_BACKUP_KEY_CHECKSUM_PAGE); -use constant INFO_BACKUP_KEY_DB_VERSION => MANIFEST_KEY_DB_VERSION; - push @EXPORT, qw(INFO_BACKUP_KEY_DB_VERSION); -use constant INFO_BACKUP_KEY_FORMAT => INI_KEY_FORMAT; - push @EXPORT, qw(INFO_BACKUP_KEY_FORMAT); -use constant INFO_BACKUP_KEY_HARDLINK => MANIFEST_KEY_HARDLINK; - push @EXPORT, qw(INFO_BACKUP_KEY_HARDLINK); -use constant INFO_BACKUP_KEY_HISTORY_ID => MANIFEST_KEY_DB_ID; - push @EXPORT, qw(INFO_BACKUP_KEY_HISTORY_ID); -use constant INFO_BACKUP_KEY_LABEL => MANIFEST_KEY_LABEL; - push @EXPORT, qw(INFO_BACKUP_KEY_LABEL); -use constant INFO_BACKUP_KEY_PRIOR => MANIFEST_KEY_PRIOR; - push @EXPORT, qw(INFO_BACKUP_KEY_PRIOR); -use constant INFO_BACKUP_KEY_REFERENCE => 'backup-reference'; - push @EXPORT, qw(INFO_BACKUP_KEY_REFERENCE); -use constant INFO_BACKUP_KEY_ONLINE => MANIFEST_KEY_ONLINE; - push @EXPORT, qw(INFO_BACKUP_KEY_ONLINE); -use constant INFO_BACKUP_KEY_SYSTEM_ID => MANIFEST_KEY_SYSTEM_ID; - push @EXPORT, qw(INFO_BACKUP_KEY_SYSTEM_ID); -use constant INFO_BACKUP_KEY_TIMESTAMP_START => MANIFEST_KEY_TIMESTAMP_START; - push @EXPORT, qw(INFO_BACKUP_KEY_TIMESTAMP_START); -use constant INFO_BACKUP_KEY_TIMESTAMP_STOP => MANIFEST_KEY_TIMESTAMP_STOP; - push @EXPORT, qw(INFO_BACKUP_KEY_TIMESTAMP_STOP); -use constant INFO_BACKUP_KEY_TYPE => MANIFEST_KEY_TYPE; - push @EXPORT, qw(INFO_BACKUP_KEY_TYPE); -use constant INFO_BACKUP_KEY_VERSION => INI_KEY_VERSION; - push @EXPORT, qw(INFO_BACKUP_KEY_VERSION); - -#################################################################################################################################### -# Global variables -#################################################################################################################################### -my $strBackupInfoMissingMsg = - FILE_BACKUP_INFO . " does not exist and is required to perform a backup.\n" . - "HINT: has a stanza-create been performed?"; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackupClusterPath, - $bRequired, - $oStorage, - $bLoad, # Should the file attemp to be loaded? - $bIgnoreMissing, # Don't error on missing files - $strCipherPassSub, # Generated passphrase to encrypt manifest files if the repo is encrypted - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strBackupClusterPath'}, - {name => 'bRequired', default => true}, - {name => 'oStorage', optional => true, default => storageRepo()}, - {name => 'bLoad', optional => true, default => true}, - {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'strCipherPassSub', optional => true}, - ); - - # Build the backup info path/file name - my $strBackupInfoFile = "${strBackupClusterPath}/" . FILE_BACKUP_INFO; - my $self = {}; - my $iResult = 0; - my $strResultMessage; - - # Init object and store variables - eval - { - $self = $class->SUPER::new( - $oStorage, $strBackupInfoFile, - {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub}); - return true; - } - or do - { - # Capture error information - $iResult = exceptionCode($EVAL_ERROR); - $strResultMessage = exceptionMessage($EVAL_ERROR); - }; - - if ($iResult != 0) - { - # If the backup info file does not exist and is required, then throw an error - # The backup info is only allowed not to exist when running a stanza-create on a new install - if ($iResult == ERROR_FILE_MISSING) - { - if ($bRequired) - { - confess &log(ERROR, "${strBackupClusterPath}/$strBackupInfoMissingMsg", ERROR_FILE_MISSING); - } - } - elsif ($iResult == ERROR_CRYPTO && $strResultMessage =~ "^unable to flush") - { - confess &log(ERROR, "unable to parse '$strBackupInfoFile'\nHINT: is or was the repo encrypted?", $iResult); - } - else - { - confess $EVAL_ERROR; - } - } - - $self->{strBackupClusterPath} = $strBackupClusterPath; - $self->{oStorage} = $oStorage; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# check -# -# Check db info and make sure it matches what is already in the repository. Return the db-id if everything matches. -#################################################################################################################################### -sub check -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $iControlVersion, - $iCatalogVersion, - $ullDbSysId, - $bRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'strDbVersion', trace => true}, - {name => 'iControlVersion', trace => true}, - {name => 'iCatalogVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - {name => 'bRequired', default => true}, - ); - - # Confirm the info file exists with the DB section - if ($bRequired) - { - $self->confirmExists(); - } - - if (!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID, undef, $ullDbSysId) || - !$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, $strDbVersion)) - { - confess &log(ERROR, "database version = ${strDbVersion}, system-id ${ullDbSysId} does not match backup version = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION) . ", system-id = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID) . "\n" . - "HINT: is this the correct stanza?", ERROR_BACKUP_MISMATCH); - } - - if (!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG, undef, $iCatalogVersion) || - !$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL, undef, $iControlVersion)) - { - confess &log(ERROR, "database control-version = ${iControlVersion}, catalog-version ${iCatalogVersion}" . - " does not match backup control-version = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL) . ", catalog-version = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG) . "\n" . - "HINT: this may be a symptom of database or repository corruption!", ERROR_BACKUP_MISMATCH); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $self->numericGet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID)} - ); -} - -#################################################################################################################################### -# add -# -# Add a backup to the info file. -#################################################################################################################################### -sub add -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oBackupManifest, - $bSave, - $bRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->add', \@_, - {name => 'oBackupManifest', trace => true}, - {name => 'bSave', default => true, trace => true}, - {name => 'bRequired', default => true, trace => true}, - ); - - # Confirm the info file exists with the DB section - if ($bRequired) - { - $self->confirmExists(); - } - - # Get the backup label - my $strBackupLabel = $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL); - - # Calculate backup sizes and references - my $lBackupSize = 0; - my $lBackupSizeDelta = 0; - my $lBackupRepoSize = 0; - my $lBackupRepoSizeDelta = 0; - my $oReferenceHash = undef; - - foreach my $strFileKey ($oBackupManifest->keys(MANIFEST_SECTION_TARGET_FILE)) - { - my $lFileSize = - $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_SIZE); - my $lRepoSize = - $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false, $lFileSize); - my $strFileReference = - $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE, false); - - # Temporary until compressed size is back in - $lBackupSize += $lFileSize; - $lBackupRepoSize += $lRepoSize; - - if (defined($strFileReference)) - { - $$oReferenceHash{$strFileReference} = true; - } - else - { - $lBackupSizeDelta += $lFileSize; - $lBackupRepoSizeDelta += $lRepoSize; - } - } - - # Set backup size info - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_SIZE, $lBackupSize); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_SIZE_DELTA, $lBackupSizeDelta); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_REPO_SIZE, $lBackupRepoSize); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA, - $lBackupRepoSizeDelta); - - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_CHECK, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_COPY, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_START, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, false)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_STOP, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, false)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_STANDBY, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_CHECKSUM_PAGE, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_CHECKSUM_PAGE)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_COMPRESS, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS)); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_FORMAT, - $oBackupManifest->numericGet(INI_SECTION_BACKREST, INI_KEY_FORMAT)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HARDLINK, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ONLINE, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE)); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TIMESTAMP_START, - $oBackupManifest->numericGet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START)); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TIMESTAMP_STOP, - $oBackupManifest->numericGet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TYPE, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_VERSION, - $oBackupManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION)); - - if ($bRequired) - { - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HISTORY_ID, - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID)); - } - # If we are reconstructing, then the history id must be taken from the manifest - else - { - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HISTORY_ID, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_ID)); - } - - if (!$oBackupManifest->test(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, CFGOPTVAL_BACKUP_TYPE_FULL)) - { - my @stryReference = sort(keys(%$oReferenceHash)); - - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_PRIOR, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_REFERENCE, - \@stryReference); - } - - if ($bSave) - { - $self->save(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# current -# -# Test if a backup is current. -#################################################################################################################################### -sub current -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup - ) = - logDebugParam - ( - __PACKAGE__ . '->current', \@_, - {name => 'strBackup'} - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bTest', value => $self->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup)} - ); -} - -#################################################################################################################################### -# list -# -# Get backup keys. -#################################################################################################################################### -sub list -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFilter, - $strOrder - ) = - logDebugParam - ( - __PACKAGE__ . '->list', \@_, - {name => 'strFilter', required => false}, - {name => 'strOrder', default => 'forward'} - ); - - # List of backups - my @stryBackup; - - # Iterate through the backups and filter - for my $strBackup ($self->keys(INFO_BACKUP_SECTION_BACKUP_CURRENT)) - { - if (!defined($strFilter) || $strBackup =~ $strFilter) - { - if ($strOrder eq 'reverse') - { - unshift(@stryBackup, $strBackup) - } - else - { - push(@stryBackup, $strBackup) - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryBackup', value => \@stryBackup} - ); -} - -#################################################################################################################################### -# backupArchiveDbHistoryId -# -# Gets the backup.info db-id for the archiveId passed. -#################################################################################################################################### -sub backupArchiveDbHistoryId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchiveId, - $strPathBackupArchive, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupArchiveDbHistoryId', \@_, - {name => 'strArchiveId'}, - {name => 'strPathBackupArchive'}, - ); - - # List of backups associated with the db-id provided - my @stryArchiveBackup; - - # Build the db list from the history in the backup info and archive info file - my $oArchiveInfo = new pgBackRestTest::Env::ArchiveInfo($strPathBackupArchive, true); - my $hDbListArchive = $oArchiveInfo->dbHistoryList(); - my $hDbListBackup = $self->dbHistoryList(); - my $iDbHistoryId = undef; - - # Get the db-version and db-id (history id) from the archiveId - my ($strDbVersionArchive, $iDbIdArchive) = split("-", $strArchiveId); - - # Get the DB system ID to map back to the backup info if it exists in the archive info file - if (exists($hDbListArchive->{$iDbIdArchive})) - { - my $ullDbSysIdArchive = $$hDbListArchive{$iDbIdArchive}{&INFO_SYSTEM_ID}; - - # Get the db-id from backup info history that corresponds to the archive db-version and db-system-id - # Sort from newest (highest db-id) to oldest - foreach my $iDbIdBackup (sort {$b <=> $a} keys %{$hDbListBackup}) - { - if ($$hDbListBackup{$iDbIdBackup}{&INFO_SYSTEM_ID} == $ullDbSysIdArchive && - $$hDbListBackup{$iDbIdBackup}{&INFO_DB_VERSION} eq $strDbVersionArchive) - { - $iDbHistoryId = $iDbIdBackup; - last; - } - } - } - - # If the database is not found in the backup.info history list - if (!defined($iDbHistoryId)) - { - # Check to see that the current DB sections match for the archive and backup info files - if (!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, - ($self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION)))) || - !($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, - ($self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID))))) - { - # This should never happen unless the backup.info file is corrupt - confess &log(ASSERT, "the archive and backup database sections do not match", ERROR_FILE_INVALID); - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $iDbHistoryId} - ); -} - -#################################################################################################################################### -# listByArchiveId -# -# Filters a list of backups by the archiveId passed. -#################################################################################################################################### -sub listByArchiveId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchiveId, - $strPathBackupArchive, - $stryBackup, - $strOrder, - ) = - logDebugParam - ( - __PACKAGE__ . '->listByArchiveId', \@_, - {name => 'strArchiveId'}, - {name => 'strPathBackupArchive'}, - {name => 'stryBackup'}, - {name => 'strOrder', default => 'forward'} - ); - - # List of backups associated with the db-id provided - my @stryArchiveBackup; - - my $iDbHistoryId = $self->backupArchiveDbHistoryId($strArchiveId, $strPathBackupArchive); - - # If history found, then build list of backups associated with the archive id passed, else return empty array - if (defined($iDbHistoryId)) - { - # Iterate through the backups and filter - foreach my $strBackup (@$stryBackup) - { - # From the backup.info current backup section, get the db-id for the backup and if it is the same, add to the list - if ($self->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_HISTORY_ID, $iDbHistoryId)) - { - if ($strOrder eq 'reverse') - { - unshift(@stryArchiveBackup, $strBackup) - } - else - { - push(@stryArchiveBackup, $strBackup) - } - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryArchiveBackup', value => \@stryArchiveBackup} - ); -} - -#################################################################################################################################### -# last -# -# Find the last backup depending on the type. -#################################################################################################################################### -sub last -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType - ) = - logDebugParam - ( - __PACKAGE__ . '->last', \@_, - {name => 'strType'} - ); - - my $strFilter = backupRegExpGet(true, $strType ne CFGOPTVAL_BACKUP_TYPE_FULL, $strType eq CFGOPTVAL_BACKUP_TYPE_INCR); - my $strBackup = ($self->list($strFilter, 'reverse'))[0]; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackup', value => $strBackup} - ); -} - -#################################################################################################################################### -# delete -# -# Delete a backup from the info file. -#################################################################################################################################### -sub delete -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackupLabel - ) = - logDebugParam - ( - __PACKAGE__ . '->delete', \@_, - {name => 'strBackupLabel'} - ); - - $self->remove(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# create -# -# Create the info file. WARNING - this file should only be called from stanza-create or test modules. -#################################################################################################################################### -sub create -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $iControlVersion, - $iCatalogVersion, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->create', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - {name => 'iControlVersion'}, - {name => 'iCatalogVersion'}, - {name => 'bSave', default => true}, - ); - - # Fill db section and db history section - $self->dbSectionSet($strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId, $self->dbHistoryIdGet(false)); - - if ($bSave) - { - $self->save(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# dbHistoryIdGet -# -# Get the db history ID -#################################################################################################################################### -sub dbHistoryIdGet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $bFileRequired, - ) = logDebugParam - ( - __PACKAGE__ . '->dbHistoryIdGet', \@_, - {name => 'bFileRequired', default => true}, - ); - - # Confirm the info file exists if it is required - if ($bFileRequired) - { - $self->confirmExists(); - } - - # If the DB section does not exist, initialize the history to one, else return the latest ID - my $iDbHistoryId = (!$self->test(INFO_BACKUP_SECTION_DB)) - ? 1 : $self->numericGet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $iDbHistoryId} - ); -} - -#################################################################################################################################### -# dbHistoryList -# -# Get the data from the db history section. -#################################################################################################################################### -sub dbHistoryList -{ - my $self = shift; - my - ( - $strOperation, - ) = logDebugParam - ( - __PACKAGE__ . '->dbHistoryList', - ); - - my %hDbHash; - - foreach my $iHistoryId ($self->keys(INFO_BACKUP_SECTION_DB_HISTORY)) - { - $hDbHash{$iHistoryId}{&INFO_DB_VERSION} = - $self->get(INFO_BACKUP_SECTION_DB_HISTORY, $iHistoryId, INFO_BACKUP_KEY_DB_VERSION); - $hDbHash{$iHistoryId}{&INFO_SYSTEM_ID} = - $self->get(INFO_BACKUP_SECTION_DB_HISTORY, $iHistoryId, INFO_BACKUP_KEY_SYSTEM_ID); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hDbHash', value => \%hDbHash} - ); -} - -#################################################################################################################################### -# dbSectionSet -# -# Set the db and db:history sections. -#################################################################################################################################### -sub dbSectionSet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $iControlVersion, - $iCatalogVersion, - $ullDbSysId, - $iDbHistoryId, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbSectionSet', \@_, - {name => 'strDbVersion', trace => true}, - {name => 'iControlVersion', trace => true}, - {name => 'iCatalogVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - {name => 'iDbHistoryId', trace => true}, - ); - - # Fill db section - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG, undef, $iCatalogVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL, undef, $iControlVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID, undef, $ullDbSysId); - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, $strDbVersion . ''); - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID, undef, $iDbHistoryId); - - # Fill db history - $self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_CATALOG, $iCatalogVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_CONTROL, $iControlVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_SYSTEM_ID, $ullDbSysId); - $self->set(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_DB_VERSION, $strDbVersion . ''); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# confirmDb -# -# Ensure that the backup is associated with the database passed. -# NOTE: The backup must exist in the backup:current section. -#################################################################################################################################### -sub confirmDb -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $strDbVersion, - $ullDbSysId, - ) = - logDebugParam - ( - __PACKAGE__ . '->confirmDb', \@_, - {name => 'strBackup', trace => true}, - {name => 'strDbVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - ); - - my $bConfirmDb = undef; - - # Get the db-id associated with the backup - my $iDbHistoryId = $self->get(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_HISTORY_ID); - - # Get the version and system-id for all known databases - my $hDbList = $self->dbHistoryList(); - - # If the db-id for the backup exists in the list - if (exists $hDbList->{$iDbHistoryId}) - { - # If the version and system-id match then database is confirmed for the backup - if (($hDbList->{$iDbHistoryId}{&INFO_DB_VERSION} eq $strDbVersion) && - ($hDbList->{$iDbHistoryId}{&INFO_SYSTEM_ID} eq $ullDbSysId)) - { - $bConfirmDb = true; - } - else - { - $bConfirmDb = false; - } - } - # If not, the backup.info file must be corrupt - else - { - confess &log(ERROR, "backup info file is missing database history information for an existing backup", ERROR_FILE_INVALID); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bConfirmDb', value => $bConfirmDb} - ); -} - -#################################################################################################################################### -# confirmExists -# -# Ensure that the backup.info file and the db section exist. -#################################################################################################################################### -sub confirmExists -{ - my $self = shift; - - # Confirm the file exists and the DB section is filled out - if (!$self->test(INFO_BACKUP_SECTION_DB) || !$self->{bExists}) - { - confess &log(ERROR, $self->{strBackupClusterPath} . "/" . $strBackupInfoMissingMsg, ERROR_FILE_MISSING); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm b/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm deleted file mode 100644 index 541bc8a2d..000000000 --- a/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm +++ /dev/null @@ -1,639 +0,0 @@ -#################################################################################################################################### -# ExpireCommonTest.pm - Common code for expire tests -#################################################################################################################################### -package pgBackRestTest::Env::ExpireEnvTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Fcntl qw(O_RDONLY); -use File::Basename qw(basename); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{oHostBackup}, - $self->{strBackRestExe}, - $self->{oStorageRepo}, - $self->{strPgPath}, - $self->{oRunTest}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oHostBackup', required => false, trace => true}, - {name => 'strBackRestExe', trace => true}, - {name => 'oStorageRepo', trace => true}, - {name => 'strPgPath', trace => true}, - {name => 'oRunTest', required => false, trace => true}, - ); - - $self->{strVm} = $self->{oRunTest}->vm(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# get into from pg_control -#################################################################################################################################### -my $oPgControlVersionHash = -{ - # iControlVersion => {iCatalogVersion => strDbVersion} - 942 => - { - 201409291 => PG_VERSION_94, - 201510051 => PG_VERSION_95, - }, - 960 => - { - 201608131 => PG_VERSION_96, - }, - 1002 => - { - 201707211 => PG_VERSION_10, - }, - 1100 => - { - 201809051 => PG_VERSION_11, - }, - 1201 => - { - 201909212 => PG_VERSION_12, - }, - 1300 => - { - 202007201 => PG_VERSION_13, - }, -}; - -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbPath - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strDbPath', default => $self->{strPgPath}} - ); - - # Open the control file and read system id and versions - #----------------------------------------------------------------------------------------------------------------------- - my $strControlFile = "${strDbPath}/" . DB_FILE_PGCONTROL; - my $hFile; - my $tBlock; - - sysopen($hFile, $strControlFile, O_RDONLY) - or confess &log(ERROR, "unable to open ${strControlFile}", ERROR_FILE_OPEN); - - # Read system identifier - sysread($hFile, $tBlock, 8) == 8 - or confess &log(ERROR, "unable to read database system identifier"); - - $self->{info}{$strDbPath}{ullDbSysId} = unpack('Q', $tBlock); - - # Read control version - sysread($hFile, $tBlock, 4) == 4 - or confess &log(ERROR, "unable to read control version"); - - $self->{info}{$strDbPath}{iDbControlVersion} = unpack('L', $tBlock); - - # Read catalog version - sysread($hFile, $tBlock, 4) == 4 - or confess &log(ERROR, "unable to read catalog version"); - - $self->{info}{$strDbPath}{iDbCatalogVersion} = unpack('L', $tBlock); - - # Close the control file - close($hFile); - - # Get PostgreSQL version - $self->{info}{$strDbPath}{strDbVersion} = - $oPgControlVersionHash->{$self->{info}{$strDbPath}{iDbControlVersion}} - {$self->{info}{$strDbPath}{iDbCatalogVersion}}; - - if (!defined($self->{info}{$strDbPath}{strDbVersion})) - { - confess &log( - ERROR, - 'unexpected control version = ' . $self->{info}{$strDbPath}{iDbControlVersion} . - ' and catalog version = ' . $self->{info}{$strDbPath}{iDbCatalogVersion} . "\n" . - 'HINT: is this version of PostgreSQL supported?'); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strDbVersion', value => $self->{info}{$strDbPath}{strDbVersion}}, - {name => 'iDbControlVersion', value => $self->{info}{$strDbPath}{iDbControlVersion}}, - {name => 'iDbCatalogVersion', value => $self->{info}{$strDbPath}{iDbCatalogVersion}}, - {name => 'ullDbSysId', value => $self->{info}{$strDbPath}{ullDbSysId}} - ); -} - -#################################################################################################################################### -# stanzaSet - set the local stanza object -#################################################################################################################################### -sub stanzaSet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strDbVersion, - $bStanzaUpgrade, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaSet', \@_, - {name => 'strStanza'}, - {name => 'strDbVersion'}, - {name => 'bStanzaUpgrade'}, - ); - - # Assign variables - my $oStanza = {}; - my $oArchiveInfo = {}; - my $oBackupInfo = {}; - my $iArchiveDbId = 1; - my $iBackupDbId = 1; - - # If we're not upgrading, then create the info files - if (!$bStanzaUpgrade) - { - $oArchiveInfo = - new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath(), false, - {bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef}); - $oBackupInfo = - new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath(), false, - {bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef}); - } - # Else get the info data from disk - else - { - $oArchiveInfo = - new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath(), - {strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef}); - $oBackupInfo = - new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath(), - {strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef}); - } - - # Get the database info for the stanza - (my $strVersion, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, $$oStanza{ullDbSysId}) = $self->info(); - $$oStanza{strDbVersion} = $strDbVersion; - - if ($bStanzaUpgrade) - { - $iArchiveDbId = $oArchiveInfo->dbHistoryIdGet() + 1; - $iBackupDbId = $oBackupInfo->dbHistoryIdGet() + 1; - } - - $oArchiveInfo->dbSectionSet($$oStanza{strDbVersion}, $$oStanza{ullDbSysId}, $iArchiveDbId); - $oArchiveInfo->save(); - - $oBackupInfo->dbSectionSet($$oStanza{strDbVersion}, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, - $$oStanza{ullDbSysId}, $iBackupDbId); - $oBackupInfo->save(); - - # Get the archive and directory paths for the stanza - $$oStanza{strArchiveClusterPath} = $self->{oHostBackup}->repoArchivePath($oArchiveInfo->archiveId()); - $$oStanza{strBackupClusterPath} = $self->{oHostBackup}->repoBackupPath(); - - $self->{oStanzaHash}{$strStanza} = $oStanza; - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaCreate -#################################################################################################################################### -sub stanzaCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strDbVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaCreate', \@_, - {name => 'strStanza'}, - {name => 'strDbVersion'}, - ); - - my $strDbVersionTemp = $strDbVersion; - $strDbVersionTemp =~ s/\.//; - - # Create the test path for pg_control - storageTest()->pathCreate(($self->{strPgPath} . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true}); - - # Generate pg_control for stanza-create - $self->controlGenerate($self->{strPgPath}, $strDbVersion); - executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL); - - # Create the stanza and set the local stanza object - $self->stanzaSet($strStanza, $strDbVersion, false); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaUpgrade -#################################################################################################################################### -sub stanzaUpgrade -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strDbVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaUpgrade', \@_, - {name => 'strStanza'}, - {name => 'strDbVersion'}, - ); - - my $strDbVersionTemp = $strDbVersion; - $strDbVersionTemp =~ s/\.//; - - # Remove pg_control - storageTest()->remove($self->{strPgPath} . '/' . DB_FILE_PGCONTROL); - - # Copy pg_control for stanza-upgrade - $self->controlGenerate($self->{strPgPath}, $strDbVersion); - executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL); - - $self->stanzaSet($strStanza, $strDbVersion, true); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} -#################################################################################################################################### -# backupCreate -#################################################################################################################################### -sub backupCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strType, - $lTimestamp, - $iArchiveBackupTotal, - $iArchiveBetweenTotal - ) = - logDebugParam - ( - __PACKAGE__ . '->backupCreate', \@_, - {name => 'strStanza'}, - {name => 'strType'}, - {name => 'lTimestamp'}, - {name => 'iArchiveBackupTotal', default => 3}, - {name => 'iArchiveBetweenTotal', default => 3} - ); - - my $oStanza = $self->{oStanzaHash}{$strStanza}; - - my ($strArchiveStart, $strArchiveStop); - - if ($iArchiveBackupTotal != -1) - { - ($strArchiveStart, $strArchiveStop) = $self->archiveCreate($strStanza, $iArchiveBackupTotal); - } - - # Create the manifest - my $oLastManifest = $strType ne CFGOPTVAL_BACKUP_TYPE_FULL ? $$oStanza{oManifest} : undef; - - my $strBackupLabel = - backupLabelFormat($strType, - defined($oLastManifest) ? $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL) : undef, - $lTimestamp); - - my $strBackupClusterSetPath = "$$oStanza{strBackupClusterPath}/${strBackupLabel}"; - - &log(INFO, "create backup ${strBackupLabel}"); - - # Get passphrase (returns undefined if repo not encrypted) to access the manifest - my $strCipherPassManifest = - (new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath()))->cipherPassSub(); - my $strCipherPassBackupSet; - - # If repo is encrypted then get passphrase for accessing the backup files from the last manifest if it exists provide one - if (defined($strCipherPassManifest)) - { - $strCipherPassBackupSet = (defined($oLastManifest)) ? $oLastManifest->cipherPassSub() : - ENCRYPTION_KEY_BACKUPSET; - } - - my $strManifestFile = "$$oStanza{strBackupClusterPath}/${strBackupLabel}/" . FILE_MANIFEST; - - my $oManifest = new pgBackRestTest::Env::Manifest($strManifestFile, {bLoad => false, strDbVersion => PG_VERSION_94, - iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94), - strCipherPass => $strCipherPassManifest, strCipherPassSub => $strCipherPassBackupSet}); - - # Store information about the backup into the backup section - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, $strBackupLabel); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK, undef, true); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY, undef, false); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, false); - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, $strArchiveStart); - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, $strArchiveStop); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, 'backup-bundle', undef, true); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_CHECKSUM_PAGE, undef, true); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, true); - $oManifest->numericSet(INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, REPOSITORY_FORMAT); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK, undef, false); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, true); - $oManifest->numericSet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef, $lTimestamp); - $oManifest->numericSet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, $lTimestamp); - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, $strType); - $oManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, PROJECT_VERSION); - - if ($strType ne CFGOPTVAL_BACKUP_TYPE_FULL) - { - if (!defined($oLastManifest)) - { - confess &log(ERROR, "oLastManifest must be defined when strType = ${strType}"); - } - - # Set backup-prior - if ($strType eq CFGOPTVAL_BACKUP_TYPE_INCR) - { - # If this is an incremental backup, then it is always based on the prior backup so use the label from the last backup - $oManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, - $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); - } - else - { - # If it is a differential then backup-prior must be set to the newest full backup so get the full backup label from - # the prior label - $oManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, - substr($oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL), 0, 16)); - } - } - - $oManifest->save(); - $$oStanza{oManifest} = $oManifest; - - # Add the backup to info - my $oBackupInfo = new pgBackRestTest::Env::BackupInfo($$oStanza{strBackupClusterPath}, false); - - $oBackupInfo->check($$oStanza{strDbVersion}, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, $$oStanza{ullDbSysId}); - $oBackupInfo->add($oManifest); - - # Create the backup description string - if (defined($$oStanza{strBackupDescription})) - { - $$oStanza{strBackupDescription} .= "\n"; - } - - $$oStanza{strBackupDescription} .= - "* ${strType} backup: label = ${strBackupLabel}" . - (defined($oLastManifest) ? ', prior = ' . $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL) : '') . - (defined($strArchiveStart) ? ", start = ${strArchiveStart}, stop = ${strArchiveStop}" : ', not online'); - - if ($iArchiveBetweenTotal != -1) - { - $self->archiveCreate($strStanza, $iArchiveBetweenTotal); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# archiveNext -#################################################################################################################################### -sub archiveNext -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchive, - $bSkipFF - ) = - logDebugParam - ( - __PACKAGE__ . '->archiveNext', \@_, - {name => 'strArchive', trace => true}, - {name => 'bSkipFF', trace => true} - ); - - # Break archive log into components - my $lTimeline = hex(substr($strArchive, 0, 8)); - my $lMajor = hex(substr($strArchive, 8, 8)); - my $lMinor = hex(substr($strArchive, 16, 8)); - - # Increment the minor component (and major when needed) - $lMinor += 1; - - if ($bSkipFF && $lMinor == 255 || !$bSkipFF && $lMinor == 256) - { - $lMajor += 1; - $lMinor = 0; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveNext', value => uc(sprintf("%08x%08x%08x", $lTimeline, $lMajor, $lMinor)), trace => true} - ); -} - -#################################################################################################################################### -# archiveCreate -#################################################################################################################################### -sub archiveCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $iArchiveTotal - ) = - logDebugParam - ( - __PACKAGE__ . '->archiveCreate', \@_, - {name => 'strStanza'}, - {name => 'iArchiveTotal'} - ); - - my $oStanza = $self->{oStanzaHash}{$strStanza}; - my $iArchiveIdx = 0; - - my $strArchive = defined($$oStanza{strArchiveLast}) ? $self->archiveNext($$oStanza{strArchiveLast}, false) : - '000000010000000000000000'; - - # Get passphrase (returns undefined if repo not encrypted) to access the archive files - my $strCipherPass = - (new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath()))->cipherPassSub(); - - push(my @stryArchive, $strArchive); - - do - { - my $strPath = "$$oStanza{strArchiveClusterPath}/" . substr($strArchive, 0, 16); - my $strFile = "${strPath}/${strArchive}-0000000000000000000000000000000000000000" . ($iArchiveIdx % 2 == 0 ? '.gz' : ''); - - storageRepo()->put($strFile, 'ARCHIVE', {strCipherPass => $strCipherPass}); - - $iArchiveIdx++; - - if ($iArchiveIdx < $iArchiveTotal) - { - $strArchive = $self->archiveNext($strArchive, false); - } - } - while ($iArchiveIdx < $iArchiveTotal); - - push(@stryArchive, $strArchive); - $$oStanza{strArchiveLast} = $strArchive; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryArchive', value => \@stryArchive} - ); -} - -#################################################################################################################################### -# process -#################################################################################################################################### -sub process -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $iExpireFull, - $iExpireDiff, - $strExpireArchiveType, - $iExpireArchive, - $strDescription - ) = - logDebugParam - ( - __PACKAGE__ . '->process', \@_, - {name => 'strStanza'}, - {name => 'iExpireFull', required => false}, - {name => 'iExpireDiff', required => false}, - {name => 'strExpireArchiveType'}, - {name => 'iExpireArchive', required => false}, - {name => 'strDescription'} - ); - - my $oStanza = $self->{oStanzaHash}{$strStanza}; - - undef($$oStanza{strBackupDescription}); - - my $strCommand = - $self->{strBackRestExe} . ' --config="' . $self->{oHostBackup}->backrestConfig() . '"' . ' --stanza=' . $strStanza . - ' --log-level-console=' . lc(DETAIL); - - if (defined($iExpireFull)) - { - $strCommand .= ' --repo1-retention-full=' . $iExpireFull; - } - - if (defined($iExpireDiff)) - { - $strCommand .= ' --repo1-retention-diff=' . $iExpireDiff; - } - - if (defined($strExpireArchiveType)) - { - if (defined($iExpireArchive)) - { - $strCommand .= ' --repo1-retention-archive-type=' . $strExpireArchiveType . - ' --repo1-retention-archive=' . $iExpireArchive; - } - else - { - $strCommand .= ' --repo1-retention-archive-type=' . $strExpireArchiveType; - } - } - - $strCommand .= ' expire'; - - $self->{oHostBackup}->executeSimple($strCommand, {strComment => $strDescription}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm b/test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm deleted file mode 100644 index d882c2275..000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm +++ /dev/null @@ -1,82 +0,0 @@ -#################################################################################################################################### -# Azure Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostAzureTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Azure defaults -#################################################################################################################################### -use constant HOST_AZURE_ACCOUNT => 'azaccount'; - push @EXPORT, qw(HOST_AZURE_ACCOUNT); -use constant HOST_AZURE_KEY => 'YXpLZXk='; - push @EXPORT, qw(HOST_AZURE_KEY); -use constant HOST_AZURE_CONTAINER => 'azcontainer'; - push @EXPORT, qw(HOST_AZURE_CONTAINER); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert"; - - my $self = $class->SUPER::new( - HOST_AZURE, 'test-' . testRunGet()->vmId() . '-' . HOST_AZURE, 'mcr.microsoft.com/azure-storage/azurite', 'root', - ["${strFakeCertPath}/s3-server.crt:/root/public.crt:ro", "${strFakeCertPath}/s3-server.key:/root/private.key:ro"], - '-e AZURITE_ACCOUNTS="' . HOST_AZURE_ACCOUNT . ':' . HOST_AZURE_KEY . '"', - 'azurite-blob --blobPort 443 --blobHost 0.0.0.0 --cert=/root/public.crt --key=/root/private.key -d debug.log" - " --disableProductStyleUrl', - false); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm deleted file mode 100644 index 6e0e0e2af..000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm +++ /dev/null @@ -1,2321 +0,0 @@ -#################################################################################################################################### -# HostBackupTest.pm - Backup host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostBackupTest; -use parent 'pgBackRestTest::Env::Host::HostBaseTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use Fcntl ':mode'; -use File::Basename qw(dirname); -use File::stat qw{lstat}; -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::Host::HostAzureTest; -use pgBackRestTest::Env::Host::HostGcsTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostS3Test; -use pgBackRestTest::Env::Host::HostSftpTest; -use pgBackRestTest::Env::Manifest; -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::VmTest; - -#################################################################################################################################### -# Error constants -#################################################################################################################################### -use constant ERROR_REPO_INVALID => 103; -push @EXPORT, qw(ERROR_REPO_INVALID); - -#################################################################################################################################### -# Latest backup link constant -#################################################################################################################################### -use constant LINK_LATEST => 'latest'; - push @EXPORT, qw(LINK_LATEST); - -#################################################################################################################################### -# Host defaults -#################################################################################################################################### -use constant HOST_PATH_LOCK => 'lock'; - push @EXPORT, qw(HOST_PATH_LOCK); -use constant HOST_PATH_LOG => 'log'; - push @EXPORT, qw(HOST_PATH_LOG); -use constant HOST_PATH_REPO => 'repo'; - -use constant HOST_PROTOCOL_TIMEOUT => 10; - push @EXPORT, qw(HOST_PROTOCOL_TIMEOUT); - -#################################################################################################################################### -# Configuration constants -#################################################################################################################################### -use constant CFGDEF_SECTION_GLOBAL => 'global'; - push @EXPORT, qw(CFGDEF_SECTION_GLOBAL); -use constant CFGDEF_SECTION_STANZA => 'stanza'; - push @EXPORT, qw(CFGDEF_SECTION_STANZA); - -use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL); -use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF); -use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR); - -use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc'; - push @EXPORT, qw(CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC); - -use constant AZURE => 'azure'; - push @EXPORT, qw(AZURE); -use constant CIFS => 'cifs'; - push @EXPORT, qw(CIFS); -use constant GCS => 'gcs'; - push @EXPORT, qw(GCS); -use constant POSIX => STORAGE_POSIX; - push @EXPORT, qw(POSIX); -use constant S3 => 's3'; - push @EXPORT, qw(S3); -use constant SFTP => 'sftp'; - push @EXPORT, qw(SFTP); - -use constant CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_DEFAULT); -use constant CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_IMMEDIATE); -use constant CFGOPTVAL_RESTORE_TYPE_NAME => 'name'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_NAME); -use constant CFGOPTVAL_RESTORE_TYPE_PRESERVE => 'preserve'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_PRESERVE); -use constant CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_STANDBY); -use constant CFGOPTVAL_RESTORE_TYPE_TIME => 'time'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_TIME); -use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_XID); - -use constant NONE => 'none'; - push @EXPORT, qw(NONE); -use constant BZ2 => 'bz2'; - push @EXPORT, qw(BZ2); -use constant GZ => 'gz'; - push @EXPORT, qw(GZ); -use constant LZ4 => 'lz4'; - push @EXPORT, qw(LZ4); -use constant ZST => 'zst'; - push @EXPORT, qw(ZST); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - # If params are not passed - my $oHostGroup = hostGroupGet(); - - my ($strName, $strImage, $strUser); - - if (!defined($$oParam{strName}) || $$oParam{strName} eq HOST_BACKUP) - { - $strName = HOST_BACKUP; - $strImage = containerRepo() . ':' . testRunGet()->vm() . '-test'; - } - else - { - $strName = $$oParam{strName}; - $strImage = $$oParam{strImage}; - } - - $strUser = testRunGet()->pgUser(); - - # Create the host - my $self = $class->SUPER::new($strName, {strImage => $strImage, strUser => $strUser, bTls => $oParam->{bTls}}); - bless $self, $class; - - # If repo is on local filesystem then set the repo-path locally - if ($oParam->{bRepoLocal} || $oParam->{strBackupDestination} eq HOST_SFTP) - { - $self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO; - } - # Else on KV store and repo will be in root - else - { - $self->{strRepoPath} = '/'; - } - - # If there is a repo2 it will always be posix on the repo host - $self->{strRepo2Path} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO . "2"; - - # Set log/lock paths - $self->{strLogPath} = $self->testPath() . '/' . HOST_PATH_LOG; - storageTest()->pathCreate($self->{strLogPath}, {strMode => '0770'}); - $self->{strLockPath} = $self->testPath() . '/' . HOST_PATH_LOCK; - - # Set conf file - $self->{strBackRestConfig} = $self->testPath() . '/' . PROJECT_CONF; - - # Set synthetic - $self->{bSynthetic} = defined($$oParam{bSynthetic}) && $$oParam{bSynthetic} ? true : false; - - # Set the backup destination - $self->{strBackupDestination} = $$oParam{strBackupDestination}; - - # Default hardlink to false - $self->{bHardLink} = false; - - # By default there is no bogus host - $self->{bBogusHost} = false; - - # Create a placeholder hash for file munging - $self->{hInfoFile} = {}; - - # Set whether repo should be encrypted or not - $self->{bRepoEncrypt} = defined($$oParam{bRepoEncrypt}) ? $$oParam{bRepoEncrypt} : false; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# timestampFileFormat -#################################################################################################################################### -sub timestampFileFormat -{ - my $strFormat = shift; - my $lTime = shift; - - return timestampFormat(defined($strFormat) ? $strFormat : '%4d%02d%02d-%02d%02d%02d', $lTime); -} - -push @EXPORT, qw(timestampFileFormat); - -#################################################################################################################################### -# backupLabelFormat -# -# Format the label for a backup. -#################################################################################################################################### -sub backupLabelFormat -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strBackupLabelLast, - $lTimestampStart - ) = - logDebugParam - ( - __PACKAGE__ . '::backupLabelFormat', \@_, - {name => 'strType', trace => true}, - {name => 'strBackupLabelLast', required => false, trace => true}, - {name => 'lTimestampTart', trace => true} - ); - - # Full backup label - my $strBackupLabel; - - if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL) - { - # Last backup label must not be defined - if (defined($strBackupLabelLast)) - { - confess &log(ASSERT, "strBackupLabelLast must not be defined when strType = '${strType}'"); - } - - # Format the timestamp and add the full indicator - $strBackupLabel = timestampFileFormat(undef, $lTimestampStart) . 'F'; - } - # Else diff or incr label - else - { - # Last backup label must be defined - if (!defined($strBackupLabelLast)) - { - confess &log(ASSERT, "strBackupLabelLast must be defined when strType = '${strType}'"); - } - - # Get the full backup portion of the last backup label - $strBackupLabel = substr($strBackupLabelLast, 0, 16); - - # Format the timestamp - $strBackupLabel .= '_' . timestampFileFormat(undef, $lTimestampStart); - - # Add the diff indicator - if ($strType eq CFGOPTVAL_BACKUP_TYPE_DIFF) - { - $strBackupLabel .= 'D'; - } - # Else incr indicator - else - { - $strBackupLabel .= 'I'; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackupLabel', value => $strBackupLabel, trace => true} - ); -} - -push @EXPORT, qw(backupLabelFormat); - -#################################################################################################################################### -# backupBegin -#################################################################################################################################### -sub backupBegin -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupBegin', \@_, - {name => 'strType', trace => true}, - {name => 'strComment', trace => true}, - {name => 'oParam', required => false, trace => true}, - ); - - # Set defaults - my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? $$oParam{oExpectedManifest} : undef; - - $strComment = - "${strType} backup" . (defined($strComment) ? " - ${strComment}" : '') . - ' (' . $self->nameGet() . ' host)'; - - &log(INFO, " $strComment"); - - # Execute the backup command - my $oExecuteBackup = $self->execute( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($oExpectedManifest) ? " --no-online" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - (defined($$oParam{bStandby}) && $$oParam{bStandby} ? " --backup-standby" : '') . - (defined($oParam->{strRepoType}) ? " --repo1-type=$oParam->{strRepoType}" : '') . - (defined($oParam->{iRepo}) ? ' --repo=' . $oParam->{iRepo} : '') . - ($strType ne 'incr' ? " --type=${strType}" : '') . - ' --stanza=' . (defined($oParam->{strStanza}) ? $oParam->{strStanza} : $self->stanza()) . ' backup', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - $oExecuteBackup->begin(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oExecuteBackup', value => $oExecuteBackup, trace => true}, - ); -} - -#################################################################################################################################### -# backupEnd -#################################################################################################################################### -sub backupEnd -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $oExecuteBackup, - $oParam, - $bManifestCompare, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupEnd', \@_, - {name => 'strType', trace => true}, - {name => 'oExecuteBackup', trace => true}, - {name => 'oParam', required => false, trace => true}, - {name => 'bManifestCompare', required => false, default => true, trace => true}, - ); - - # Set defaults - my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? dclone($$oParam{oExpectedManifest}) : undef; - - my $iExitStatus = $oExecuteBackup->end(); - - return if ($oExecuteBackup->{iExpectedExitStatus} != 0); - - # If an alternate stanza was specified - if (defined($oParam->{strStanza})) - { - confess &log(ASSERT, - 'if an alternate stanza is specified it must generate an error - the remaining code will not be aware of the stanza'); - } - - my $strBackup = $self->backupLast($oParam->{iRepo}); - - # Only compare backups that are in repo1. There is not a lot of value in comparing backups in other repos and it would require a - # lot of changes to the test harness. - if (!defined($oParam->{iRepo}) || $oParam->{iRepo} == 1) - { - # If a real backup then load the expected manifest from the actual manifest. An expected manifest can't be generated - # perfectly because a running database is always in flux. Even so, it allows us to test many things. - if (!$self->synthetic()) - { - $oExpectedManifest = iniParse( - ${storageRepo()->get( - storageRepo()->openRead( - 'backup/' . $self->stanza() . "/${strBackup}/" . FILE_MANIFEST, - {strCipherPass => $self->cipherPassManifest()}))}); - } - - # Make sure tablespace links are correct - if ($self->hasLink()) - { - if (($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink()) && - !$oExpectedManifest->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'}) - { - my $hTablespaceManifest = storageTest()->manifest( - $self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)); - - # Remove . and .. - delete($hTablespaceManifest->{'.'}); - delete($hTablespaceManifest->{'..'}); - - # Iterate file links - for my $strFile (sort(keys(%{$hTablespaceManifest}))) - { - # Make sure the link is in the expected manifest - my $hManifestTarget = - $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"}; - - if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK || - $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile) - { - confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id"); - } - - # Make sure the link really is a link - if ($hTablespaceManifest->{$strFile}{type} ne 'l') - { - confess &log(ERROR, "'${strFile}' in tablespace directory is not a link"); - } - - # Make sure the link destination is correct - my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}"; - - if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination) - { - confess &log(ERROR, - "'${strFile}' link should reference '${strLinkDestination}' but actually references " . - "'$hTablespaceManifest->{$strFile}{link_destination}'"); - } - } - - # Iterate manifest targets - for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}}))) - { - my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}; - my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID}; - - # Make sure the target exists as a link on disk - if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) && - !defined($hTablespaceManifest->{$strTablespaceId})) - { - confess &log(ERROR, - "target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'"); - } - } - } - # Else there should not be a tablespace directory at all. This is only valid for storage that supports links. - elsif (storageRepo()->capability(STORAGE_CAPABILITY_LINK) && - storageTest()->pathExists( - $self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))) - { - confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory'); - } - } - - # Check that latest link exists unless repo links are disabled - my $strLatestLink = $self->repoBackupPath(LINK_LATEST); - my $bLatestLinkExists = storageRepo()->exists($strLatestLink); - - if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq POSIX) && $self->hasLink()) - { - my $strLatestLinkDestination = readlink($strLatestLink); - - if ($strLatestLinkDestination ne $strBackup) - { - confess &log(ERROR, "'" . LINK_LATEST . "' link should be '${strBackup}' but is '${strLatestLinkDestination}"); - } - } - elsif ($bLatestLinkExists) - { - confess &log(ERROR, "'" . LINK_LATEST . "' link should not exist"); - } - - # Only do compare for synthetic backups since for real backups the expected manifest *is* the actual manifest. - if ($self->synthetic()) - { - # Compare only if expected to do so - if ($bManifestCompare) - { - # Set backup type in the expected manifest - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE} = $strType; - - $self->backupCompare($strBackup, $oExpectedManifest); - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackup', value => $strBackup, trace => true}, - ); -} - -#################################################################################################################################### -# backup -#################################################################################################################################### -sub backup -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strComment, - $oParam, - $bManifestCompare, - ) = - logDebugParam - ( - __PACKAGE__ . '->backup', \@_, - {name => 'strType'}, - {name => 'strComment'}, - {name => 'oParam', required => false}, - {name => 'bManifestCompare', required => false, default => true}, - ); - - my $oExecuteBackup = $self->backupBegin($strType, $strComment, $oParam); - my $strBackup = $self->backupEnd($strType, $oExecuteBackup, $oParam, $bManifestCompare); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackup', value => $strBackup}, - ); -} - -#################################################################################################################################### -# backupCompare -#################################################################################################################################### -sub backupCompare -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $oExpectedManifest, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupCompare', \@_, - {name => 'strBackup', trace => true}, - {name => 'oExpectedManifest', trace => true}, - ); - - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup; - - my $oActualManifest = new pgBackRestTest::Env::Manifest( - $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()}); - - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{'backup-reference'} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, 'backup-reference'); - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START); - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP); - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START); - ${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = - $oActualManifest->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM); - ${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = REPOSITORY_FORMAT + 0; - - if (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && - defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}) && - $oActualManifest->test(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS)) - { - $oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS} = - $oActualManifest->get(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS); - } - - # Update the expected manifest with whether the --delta option was used or not to perform the backup. - $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = - $oActualManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA) ? INI_TRUE : INI_FALSE; - - my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); - - foreach my $strFileKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # Remove repo checksum - $oActualManifest->remove(&MANIFEST_SECTION_TARGET_FILE, $strFileKey, 'rck'); - - # Determine repo size if compression or encryption is enabled - my $strCompressType = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}; - - if ($strCompressType ne NONE || - (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && - defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}))) - { - - my $lRepoSize = - $oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ? - $oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) : - (storageRepo()->info( - $self->repoBackupPath("${strBackup}/${strFileKey}") . - ($strCompressType eq NONE ? '' : ".${strCompressType}")))->{size}; - - if (defined($lRepoSize) && - $lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE}) - { - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} = $lRepoSize; - } - } - - # If the backup does not have page checksums then no need to compare - if (!$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}) - { - delete($oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}); - delete($oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); - } - # Else make sure things that should have checks do have checks - elsif ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_CHECKSUM_PAGE) != - isChecksumPage($strFileKey)) - { - confess - "check-page actual for ${strFileKey} is " . - ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, - MANIFEST_SUBKEY_CHECKSUM_PAGE) ? 'set' : '[undef]') . - ' but isChecksumPage() says it should be ' . - (isChecksumPage($strFileKey) ? 'set' : 'undef') . '.'; - } - } - - $self->manifestDefault($oExpectedManifest); - - my $strTestPath = $self->testPath(); - - storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent})); - storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifest)); - - executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest"); - - storageTest()->remove("${strTestPath}/expected.manifest"); - storageTest()->remove("${strTestPath}/actual.manifest"); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# manifestDefault -#################################################################################################################################### -sub manifestDefault -{ - my $self = shift; - my $oExpectedManifest = shift; - - # Defaults for subkeys that tend to repeat - my $strDefaultUser = $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_USER}; - my $strDefaultGroup = $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_GROUP}; - my $strDefaultPathMode = $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_MODE}; - my $strDefaultFileMode = sprintf('%04o', oct($strDefaultPathMode) & (S_IRUSR | S_IWUSR | S_IRGRP)); - - # Remove subkeys that match the defaults - foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK) - { - next if !defined($oExpectedManifest->{$strSection}); - - foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}})) - { - if (defined($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_USER}) && - $oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_USER} eq $strDefaultUser) - { - delete($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_USER}); - } - - if (defined($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_GROUP}) && - $oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_GROUP} eq $strDefaultGroup) - { - delete($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_GROUP}); - } - - if (defined($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_MODE}) && - $oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_MODE} eq - ($strSection eq MANIFEST_SECTION_TARGET_PATH ? $strDefaultPathMode : $strDefaultFileMode)) - { - delete($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_MODE}); - } - } - } - - # Write defaults - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE . ':default'}{&MANIFEST_SUBKEY_USER} = $strDefaultUser; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE . ':default'}{&MANIFEST_SUBKEY_GROUP} = $strDefaultGroup; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE . ':default'}{&MANIFEST_SUBKEY_MODE} = $strDefaultFileMode; - - if (defined($oExpectedManifest->{&MANIFEST_SECTION_TARGET_LINK})) - { - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_LINK . ':default'}{&MANIFEST_SUBKEY_USER} = $strDefaultUser; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_LINK . ':default'}{&MANIFEST_SUBKEY_GROUP} = $strDefaultGroup; - } - - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH . ':default'}{&MANIFEST_SUBKEY_USER} = $strDefaultUser; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH . ':default'}{&MANIFEST_SUBKEY_GROUP} = $strDefaultGroup; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH . ':default'}{&MANIFEST_SUBKEY_MODE} = $strDefaultPathMode; -} - -#################################################################################################################################### -# backupLast -#################################################################################################################################### -sub backupLast -{ - my $self = shift; - my $iRepo = shift; - - my @stryBackup = storageRepo({iRepo => $iRepo})->list( - $self->repoBackupPath(undef, $iRepo), - {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'}); - - if (!defined($stryBackup[0])) - { - confess 'no backup was found: ' . join(@stryBackup, ', '); - } - - return $stryBackup[0]; -} - -#################################################################################################################################### -# check -#################################################################################################################################### -sub check -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'check ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{iTimeout}) ? " --archive-timeout=$$oParam{iTimeout}" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - (!defined($oParam->{bStanza}) || $oParam->{bStanza} ? ' --stanza=' . $self->stanza() : '') . ' check', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# expire -#################################################################################################################################### -sub expire -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'oParam', required => false}, - ); - - my $strComment = - 'expire' . - (defined($$oParam{iRetentionFull}) ? " full=$$oParam{iRetentionFull}" : '') . - (defined($$oParam{iRetentionDiff}) ? " diff=$$oParam{iRetentionDiff}" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " ${strComment}"); - - # Determine whether or not to expect an error - my $oHostGroup = hostGroupGet(); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{iRetentionFull}) ? " --repo1-retention-full=$$oParam{iRetentionFull}" : '') . - (defined($$oParam{iRetentionDiff}) ? " --repo1-retention-diff=$$oParam{iRetentionDiff}" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' --repo=' . (defined($oParam->{iRepo}) ? $oParam->{iRepo} : '1') . - ' --stanza=' . $self->stanza() . ' expire', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); -} - -#################################################################################################################################### -# info -#################################################################################################################################### -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'info' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --log-level-console=warn' . - (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . - (defined($$oParam{strOutput}) ? " --output=$$oParam{strOutput}" : '') . ' info', - {strComment => $strComment, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaCreate -#################################################################################################################################### -sub stanzaCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaCreate', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'stanza-create ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --stanza=' . $self->stanza() . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' stanza-create', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - if (storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO)) - { - # Get the passphrase for accessing the manifest file - $self->{strCipherPassManifest} = (new pgBackRestTest::Env::BackupInfo($self->repoBackupPath()))->cipherPassSub(); - } - - if (storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE)) - { - - # Get the passphrase for accessing the archived files - $self->{strCipherPassArchive} = - (new pgBackRestTest::Env::ArchiveInfo($self->repoArchivePath()))->cipherPassSub(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaUpgrade -#################################################################################################################################### -sub stanzaUpgrade -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaUpgrade', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'stanza-upgrade ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --stanza=' . $self->stanza() . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' stanza-upgrade', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaDelete -#################################################################################################################################### -sub stanzaDelete -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaDelete', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'stanza-delete ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --repo=' . (defined($oParam->{iRepo}) ? $oParam->{iRepo} : '1') . - ' --stanza=' . $self->stanza() . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' stanza-delete', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# start -#################################################################################################################################### -sub start -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->start', \@_, - {name => 'oParam', required => false}, - ); - - my $strComment = - 'start' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . ' start', - {strComment => $strComment, bLogOutput => $self->synthetic()}); -} - -#################################################################################################################################### -# stop -#################################################################################################################################### -sub stop -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stop', \@_, - {name => 'oParam', required => false}, - ); - - my $strComment = - 'stop' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . - (defined($$oParam{bForce}) && $$oParam{bForce} ? ' --force' : '') . ' stop', - {strComment => $strComment, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# configCreate -#################################################################################################################################### -sub configCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stop', \@_, - {name => 'oParam', required => false}, - ); - - my %oParamHash; - my $strStanza = $self->stanza(); - my $oHostGroup = hostGroupGet(); - my $oHostBackup = $oHostGroup->hostGet($self->backupDestination()); - my $oHostDbPrimary = $oHostGroup->hostGet(HOST_DB_PRIMARY); - my $oHostDbStandby = $oHostGroup->hostGet(HOST_DB_STANDBY, true); - - my $bArchiveAsync = defined($$oParam{bArchiveAsync}) ? $$oParam{bArchiveAsync} : false; - - my $iRepoTotal = defined($oParam->{iRepoTotal}) ? $oParam->{iRepoTotal} : 1; - - if ($iRepoTotal < 1 || $iRepoTotal > 2) - { - confess "invalid repo total ${iRepoTotal}"; - } - - # General options - # ------------------------------------------------------------------------------------------------------------------------------ - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'beta'} = 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'job-retry'} = 0; - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-console'} = lc(DETAIL); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-file'} = testRunGet()->logLevelTestFile(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-stderr'} = lc(OFF); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-subprocess'} = - testRunGet()->logLevelTestFile() eq lc(OFF) ? 'n' : 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-timestamp'} = 'n'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'buffer-size'} = '64k'; - - if ($oParam->{bBundle}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-bundle'} = 'y'; - # Set bundle size/limit smaller for testing and because FakeGCS does not do multi-part upload - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-bundle-size'} = '1MiB'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-bundle-limit'} = '64KiB'; - } - - if ($oParam->{bBlockIncr}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-block'} = 'y'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'protocol-timeout'} = 60; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'db-timeout'} = 45; - - # Set to make sure that changing the default works and to speed compression for testing - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level'} = 3; - - # Only set network compress level if there is more than one host - if ($oHostBackup != $oHostDbPrimary) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level-network'} = 1; - } - - if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne 'gz') - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-type'} = $oParam->{strCompressType}; - } - - if ($self->isHostBackup()) - { - if ($self->repoEncrypt()) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-type'} = CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-pass'} = 'x'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath(); - - # S3 settings - if ($oParam->{strStorage} eq S3) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = S3; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key'} = HOST_S3_ACCESS_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key-secret'} = HOST_S3_ACCESS_SECRET_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-bucket'} = HOST_S3_BUCKET; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-endpoint'} = HOST_S3_ENDPOINT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-region'} = HOST_S3_REGION; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'} = 'n'; - } - elsif ($oParam->{strStorage} eq AZURE) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = AZURE; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-account'} = HOST_AZURE_ACCOUNT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-key'} = HOST_AZURE_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-container'} = HOST_AZURE_CONTAINER; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-host'} = HOST_AZURE; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-verify-tls'} = 'n'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-uri-style'} = 'path'; - } - elsif ($oParam->{strStorage} eq GCS) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = GCS; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-bucket'} = HOST_GCS_BUCKET; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-key-type'} = HOST_GCS_KEY_TYPE; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-key'} = HOST_GCS_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-endpoint'} = HOST_GCS . ':' . HOST_GCS_PORT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-storage-verify-tls'} = 'n'; - } - - if ($iRepoTotal == 2) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-path'} = $self->repo2Path(); - } - - if (defined($$oParam{bHardlink}) && $$oParam{bHardlink}) - { - $self->{bHardLink} = true; - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'repo1-s3-hardlink'} = 'y'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'archive-copy'} = 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y'; - } - - # Host specific options - # ------------------------------------------------------------------------------------------------------------------------------ - - # If this is the backup host - if ($self->isHostBackup()) - { - my $oHostDb1 = $oHostDbPrimary; - my $oHostDb2 = $oHostDbStandby; - - if ($self->nameTest(HOST_DB_STANDBY)) - { - $oHostDb1 = $oHostDbStandby; - $oHostDb2 = $oHostDbPrimary; - } - - if ($self->nameTest(HOST_BACKUP)) - { - $oParamHash{$strStanza}{'pg1-host'} = $oHostDb1->nameGet(); - $oParamHash{$strStanza}{'pg1-host-user'} = $oHostDb1->userGet(); - $oParamHash{$strStanza}{'pg1-host-cmd'} = $oHostDb1->backrestExe(); - $oParamHash{$strStanza}{'pg1-host-config'} = $oHostDb1->backrestConfig(); - - if ($oParam->{bTls}) - { - $oParamHash{$strStanza}{'pg1-host-type'} = 'tls'; - $oParamHash{$strStanza}{'pg1-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{$strStanza}{'pg1-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - - # Port can't be configured for a synthetic host - if (!$self->synthetic()) - { - $oParamHash{$strStanza}{'pg1-port'} = $oHostDb1->pgPort(); - } - } - - $oParamHash{$strStanza}{'pg1-path'} = $oHostDb1->dbBasePath(); - - if (defined($oHostDb2)) - { - # Add an invalid replica to simulate more than one replica. A warning should be thrown when a stanza is created and a - # valid replica should be chosen. Only do this for SSH since TLS takes longer to timeout. - if (!$oParam->{bTls}) - { - $oParamHash{$strStanza}{"pg2-host"} = BOGUS; - $oParamHash{$strStanza}{"pg2-host-user"} = $oHostDb2->userGet(); - $oParamHash{$strStanza}{"pg2-host-cmd"} = $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{"pg2-host-config"} = $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{"pg2-path"} = $oHostDb2->dbBasePath(); - } - - # Set a flag so we know there's a bogus host - $self->{bBogusHost} = true; - - # Set a valid replica to a higher index to ensure skipping indexes does not make a difference - $oParamHash{$strStanza}{"pg256-host"} = $oHostDb2->nameGet(); - $oParamHash{$strStanza}{"pg256-host-user"} = $oHostDb2->userGet(); - $oParamHash{$strStanza}{"pg256-host-cmd"} = $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{"pg256-host-config"} = $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{"pg256-path"} = $oHostDb2->dbBasePath(); - - if ($oParam->{bTls}) - { - $oParamHash{$strStanza}{'pg256-host-type'} = 'tls'; - $oParamHash{$strStanza}{'pg256-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{$strStanza}{'pg256-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - - # Only test explicit ports on the backup server. This is so locally configured ports are also tested. - if (!$self->synthetic() && $self->nameTest(HOST_BACKUP)) - { - $oParamHash{$strStanza}{"pg256-port"} = $oHostDb2->pgPort(); - } - } - } - elsif ($oParam->{strStorage} eq SFTP) - { - my $oHostDb1 = $oHostDbPrimary; - my $oHostDb2 = $oHostDbStandby; - - if ($self->nameTest(HOST_DB_STANDBY)) - { - $oHostDb1 = $oHostDbStandby; - $oHostDb2 = $oHostDbPrimary; - } - - # Set a flag so we know there's a bogus host - $self->{bBogusHost} = true; - - # Set a valid replica to a higher index to ensure skipping indexes does not make a difference - $oParamHash{$strStanza}{"pg256-host"} = $oHostDb2->nameGet(); - $oParamHash{$strStanza}{"pg256-host-user"} = $oHostDb2->userGet(); - $oParamHash{$strStanza}{"pg256-host-cmd"} = $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{"pg256-host-config"} = $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{"pg256-path"} = $oHostDb2->dbBasePath(); - $oParamHash{$strStanza}{"pg256-port"} = $oHostDb2->pgPort(); - } - - # If this is a database host - if ($self->isHostDb()) - { - $oParamHash{$strStanza}{'pg1-path'} = $self->dbBasePath(); - - if (!$self->synthetic()) - { - $oParamHash{$strStanza}{'pg1-socket-path'} = $self->pgSocketPath(); - $oParamHash{$strStanza}{'pg1-port'} = $self->pgPort(); - } - - if ($bArchiveAsync) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':archive-push'}{'archive-async'} = 'y'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'spool-path'} = $self->spoolPath(); - - # If the backup host is remote - if (!$self->isHostBackup()) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-user'} = $oHostBackup->userGet(); - - if ($oHostBackup->nameGet() eq HOST_SFTP) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = "sftp"; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host'} = HOST_SFTP; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host-key-hash-type'} = "sha1"; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host-user'} = TEST_USER; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-private-key-file'} = testRunGet()->basePath() . SSH_PRIVATE_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-public-key-file'} = testRunGet()->basePath() . SSH_PUBLIC_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host-key-check-type'} = "none"; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath(); - - # At what count do we hit diminishing returns - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'process-max'} = 8; - - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y'; - } - else - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host'} = $oHostBackup->nameGet(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cmd'} = $oHostBackup->backrestExe(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-config'} = $oHostBackup->backrestConfig(); - } - - if ($oParam->{bTls}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-type'} = 'tls'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - - if ($iRepoTotal == 2) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host'} = $oHostBackup->nameGet(); - $oParam->{bTls} ? $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-type'} = 'tls' : undef; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-user'} = $oHostBackup->userGet(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-cmd'} = $oHostBackup->backrestExe(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-config'} = $oHostBackup->backrestConfig(); - - if ($oParam->{bTls}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-type'} = 'tls'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); - } - } - - # Write out the configuration file - storageTest()->put($self->backrestConfig(), iniRender(\%oParamHash, true)); -} - -#################################################################################################################################### -# configUpdate - update configuration with new options -#################################################################################################################################### -sub configUpdate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $hParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->configUpdate', \@_, - {name => 'hParam'}, - ); - - # Load db config file - my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true}); - - # Load params - foreach my $strSection (keys(%{$hParam})) - { - foreach my $strKey (keys(%{$hParam->{$strSection}})) - { - if (defined($hParam->{$strSection}{$strKey})) - { - $oConfig->{$strSection}{$strKey} = $hParam->{$strSection}{$strKey}; - } - else - { - delete($oConfig->{$strSection}{$strKey}); - } - } - } - - storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# manifestMunge -# -# Allows for munging of the manifest while making it appear to be valid. This is used to create various error conditions that should -# be caught by the unit tests. -#################################################################################################################################### -sub manifestMunge -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $hParam, - $bCache, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifestMunge', \@_, - {name => 'strBackup'}, - {name => '$hParam'}, - {name => 'bCache', default => true}, - ); - - $self->infoMunge($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# manifestRestore -#################################################################################################################################### -sub manifestRestore -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifestRestore', \@_, - {name => 'strBackup'}, - {name => 'bSave', default => true}, - ); - - $self->infoRestore($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $bSave); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# infoMunge -# -# With the file name specified (e.g. /repo/archive/db/archive.info) copy the current values from the file into the global hash and -# update the file with the new values passed. Later, using infoRestore, the global variable will be used to restore the file to its -# original state. -#################################################################################################################################### -sub infoMunge -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileName, - $hParam, - $bCache, - $bManifest, - ) = - logDebugParam - ( - __PACKAGE__ . '->infoMunge', \@_, - {name => 'strFileName'}, - {name => 'hParam'}, - {name => 'bCache', default => true}, - {name => 'bManifest', default => false}, - ); - - # If the original file content does not exist then load it - if (!defined($self->{hInfoFile}{$strFileName})) - { - $self->{hInfoFile}{$strFileName} = new pgBackRestDoc::Common::Ini( - storageRepo(), $strFileName, - {strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); - } - - # Make a copy of the original file contents - my $oMungeIni = new pgBackRestDoc::Common::Ini( - storageRepo(), $strFileName, - {bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), - strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); - - # Load params - foreach my $strSection (keys(%{$hParam})) - { - foreach my $strKey (keys(%{$hParam->{$strSection}})) - { - if (ref($hParam->{$strSection}{$strKey}) eq 'HASH') - { - foreach my $strSubKey (keys(%{$hParam->{$strSection}{$strKey}})) - { - # Munge the copy with the new parameter values - if (defined($hParam->{$strSection}{$strKey}{$strSubKey})) - { - $oMungeIni->set($strSection, $strKey, $strSubKey, $hParam->{$strSection}{$strKey}{$strSubKey}); - } - else - { - $oMungeIni->remove($strSection, $strKey, $strSubKey); - } - } - } - else - { - # Munge the copy with the new parameter values - if (defined($hParam->{$strSection}{$strKey})) - { - $oMungeIni->set($strSection, $strKey, undef, $hParam->{$strSection}{$strKey}); - } - else - { - $oMungeIni->remove($strSection, $strKey); - } - } - } - } - - # Save the munged data to the file - $oMungeIni->save(); - - # Clear the cache is requested - if (!$bCache) - { - delete($self->{hInfoFile}{$strFileName}); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# infoRestore -# -# With the file name specified (e.g. /repo/archive/db/archive.info) use the original file contents in the global hash to restore the -# file to its original state after modifying the values with infoMunge. -#################################################################################################################################### -sub infoRestore -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileName, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->infoRestore', \@_, - {name => 'strFileName'}, - {name => 'bSave', default => true}, - ); - - # If the original file content exists in the global hash, then save it to the file - if (defined($self->{hInfoFile}{$strFileName})) - { - if ($bSave) - { - # Save the munged data to the file - $self->{hInfoFile}{$strFileName}->{bModified} = true; - $self->{hInfoFile}{$strFileName}->save(); - } - } - else - { - confess &log(ASSERT, "There is no original data cached for $strFileName. infoMunge must be called first."); - } - - # Remove the element from the hash - delete($self->{hInfoFile}{$strFileName}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# configRecovery -#################################################################################################################################### -sub configRecovery -{ - my $self = shift; - my $oHostBackup = shift; - my $oRecoveryHashRef = shift; - - # Get stanza - my $strStanza = $self->stanza(); - - # Load db config file - my $oConfig = iniParse(${storageTest->get($self->backrestConfig())}, {bRelaxed => true}); - - # Rewrite recovery options - my @stryRecoveryOption; - - foreach my $strOption (sort(keys(%$oRecoveryHashRef))) - { - push (@stryRecoveryOption, "${strOption}=${$oRecoveryHashRef}{$strOption}"); - } - - if (@stryRecoveryOption) - { - $oConfig->{$strStanza}{'recovery-option'} = \@stryRecoveryOption; - } - - # Save db config file - storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); -} - -#################################################################################################################################### -# configRemap -#################################################################################################################################### -sub configRemap -{ - my $self = shift; - my $oRemapHashRef = shift; - my $oManifestRef = shift; - - # Get stanza name - my $strStanza = $self->stanza(); - - # Load db config file - my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true}); - - # Load backup config file - my $oRemoteConfig; - my $oHostBackup = - !$self->standby() && !$self->nameTest($self->backupDestination()) ? - hostGroupGet()->hostGet($self->backupDestination()) : undef; - - if (defined($oHostBackup)) - { - $oRemoteConfig = iniParse(${storageTest()->get($oHostBackup->backrestConfig())}, {bRelaxed => true}); - } - - # Rewrite recovery section - delete($oConfig->{"${strStanza}:restore"}{'tablespace-map'}); - my @stryTablespaceMap; - - foreach my $strRemap (sort(keys(%$oRemapHashRef))) - { - my $strRemapPath = ${$oRemapHashRef}{$strRemap}; - - if ($strRemap eq MANIFEST_TARGET_PGDATA) - { - $oConfig->{$strStanza}{'pg1-path'} = $strRemapPath; - - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; - - if (defined($oHostBackup)) - { - $oRemoteConfig->{$strStanza}{'pg1-path'} = $strRemapPath; - } - } - else - { - my $strTablespaceOid = (split('\/', $strRemap))[1]; - push (@stryTablespaceMap, "${strTablespaceOid}=${strRemapPath}"); - - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strRemap}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{MANIFEST_TARGET_PGDATA . "/${strRemap}"}{destination} = $strRemapPath; - } - } - - if (@stryTablespaceMap) - { - $oConfig->{"${strStanza}:restore"}{'tablespace-map'} = \@stryTablespaceMap; - } - - # Save db config file - storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); - - # Save backup config file (but not if this is the standby which is not the source of backups) - if (defined($oHostBackup)) - { - storageTest()->put($oHostBackup->backrestConfig(), iniRender($oRemoteConfig, true)); - } -} - -#################################################################################################################################### -# restore -#################################################################################################################################### -sub restore -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $strBackup, - $rhExpectedManifest, - $rhRemapHash, - $bDelta, - $bForce, - $strType, - $strTarget, - $bTargetExclusive, - $strTargetAction, - $strTargetTimeline, - $rhRecoveryHash, - $iExpectedExitStatus, - $strOptionalParam, - $bTablespace, - $strUser, - $strBackupExpected, - $iRepo, - ) = - logDebugParam - ( - __PACKAGE__ . '->restore', \@_, - {name => 'strComment', required => false}, - {name => 'strBackup'}, - {name => 'rhExpectedManifest', optional => true}, - {name => 'rhRemapHash', optional => true}, - {name => 'bDelta', optional => true, default => false}, - {name => 'bForce', optional => true, default => false}, - {name => 'strType', optional => true}, - {name => 'strTarget', optional => true}, - {name => 'bTargetExclusive', optional => true, default => false}, - {name => 'strTargetAction', optional => true}, - {name => 'strTargetTimeline', optional => true}, - {name => 'rhRecoveryHash', optional => true}, - {name => 'iExpectedExitStatus', optional => true}, - {name => 'strOptionalParam', optional => true}, - {name => 'bTablespace', optional => true}, - {name => 'strUser', optional => true}, - {name => 'strBackupExpected', optional => true}, - {name => 'iRepo', optional => true}, - ); - - # Build link map options - my $strLinkMap; - - foreach my $strTarget (sort(keys(%{$self->{hLinkRemap}}))) - { - $strLinkMap .= " --link-map=\"${strTarget}=${$self->{hLinkRemap}}{$strTarget}\""; - } - - $strComment = 'restore' . - ($bDelta ? ' delta' : '') . - ($bForce ? ', force' : '') . - ($strBackup ne 'latest' ? ", backup '${strBackup}'" : '') . - # This does not output 'default' for synthetic tests to make expect logs match up (may change later) - ($strType ? ", type '${strType}'" : (defined($rhExpectedManifest) ? '' : ", type 'default'")) . - ($strTarget ? ", target '${strTarget}'" : '') . - ($strTargetTimeline ? ", timeline '${strTargetTimeline}'" : '') . - ($bTargetExclusive ? ', exclusive' : '') . - (defined($strTargetAction) && $strTargetAction ne 'pause' ? ", target-action=${strTargetAction}" : '') . - (defined($rhRemapHash) ? ', remap' : '') . - (defined($iExpectedExitStatus) ? ", expect exit ${iExpectedExitStatus}" : '') . - (defined($strComment) ? " - ${strComment}" : '') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " ${strComment}"); - - # Get the backup host - my $oHostGroup = hostGroupGet(); - my $oHostBackup = defined($oHostGroup->hostGet(HOST_BACKUP, true)) ? $oHostGroup->hostGet(HOST_BACKUP) : $self; - - # If the repo was not passed, then use repo1 as the repo for getting the expected manifest/backup - my $iRepoDefault = !defined($iRepo) ? 1 : $iRepo; - - # Load the expected manifest if it was not defined - my $oExpectedManifest = undef; - - # If an expected backup is defined, then the strBackup should be the default to allow the restore process to select the backup - # - which should be the backup passed as strBackupExpected. If it is not defined, then set it based on the strBackup passed. - if (!defined($strBackupExpected)) - { - $strBackupExpected = $strBackup eq 'latest' ? $oHostBackup->backupLast($iRepoDefault) : $strBackup; - } - - if (!defined($rhExpectedManifest)) - { - # Load the manifest from the backup expected to be chosen/processed by restore - my $oExpectedManifest = new pgBackRestTest::Env::Manifest( - $self->repoBackupPath($strBackupExpected . qw{/} . FILE_MANIFEST, $iRepoDefault), - {strCipherPass => $oHostBackup->cipherPassManifest(), oStorage => storageRepo({iRepo => $iRepoDefault})}); - - $rhExpectedManifest = $oExpectedManifest->{oContent}; - - # Remap links in the expected manifest - foreach my $strTarget (sort(keys(%{$self->{hLinkRemap}}))) - { - my $strDestination = ${$self->{hLinkRemap}}{$strTarget}; - my $strTarget = 'pg_data/' . $strTarget; - my $strTargetPath = $strDestination; - - # If this link is to a file then the specified path must be split into file and path parts - if ($oExpectedManifest->isTargetFile($strTarget)) - { - $strTargetPath = dirname($strTargetPath); - - # Error when the path is not deep enough to be valid - if (!defined($strTargetPath)) - { - confess &log(ERROR, "${strDestination} is not long enough to be target for ${strTarget}"); - } - - # Set the file part - $oExpectedManifest->set( - MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE, - substr($strDestination, length($strTargetPath) + 1)); - - # Set the link target - $oExpectedManifest->set( - MANIFEST_SECTION_TARGET_LINK, $strTarget, MANIFEST_SUBKEY_DESTINATION, $strDestination); - } - else - { - # Set the link target - $oExpectedManifest->set(MANIFEST_SECTION_TARGET_LINK, $strTarget, MANIFEST_SUBKEY_DESTINATION, $strTargetPath); - } - - # Set the target path - $oExpectedManifest->set(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_PATH, $strTargetPath); - } - } - - # Get the backup host - if (defined($rhRemapHash)) - { - $self->configRemap($rhRemapHash, $rhExpectedManifest); - } - - if (defined($rhRecoveryHash)) - { - $self->configRecovery($oHostBackup, $rhRecoveryHash); - } - - # Create the restore command - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ($bDelta ? ' --delta' : '') . - ($bForce ? ' --force' : '') . - ($strBackup ne 'latest' ? " --set=${strBackup}" : '') . - (defined($strOptionalParam) ? " ${strOptionalParam} " : '') . - (defined($strType) && $strType ne CFGOPTVAL_RESTORE_TYPE_DEFAULT ? " --type=${strType}" : '') . - (defined($strTarget) ? " --target=\"${strTarget}\"" : '') . - (defined($strTargetTimeline) ? " --target-timeline=\"${strTargetTimeline}\"" : '') . - ($bTargetExclusive ? ' --target-exclusive' : '') . - (defined($strLinkMap) ? $strLinkMap : '') . - ($self->synthetic() ? '' : ' --link-all') . - (defined($strTargetAction) && $strTargetAction ne 'pause' ? " --target-action=${strTargetAction}" : '') . - (defined($iRepo) ? " --repo=${iRepo}" : '') . - " --stanza=" . $self->stanza() . ' restore', - {strComment => $strComment, iExpectedExitStatus => $iExpectedExitStatus, bLogOutput => $self->synthetic()}, - $strUser); - - if (!defined($iExpectedExitStatus)) - { - # Only compare restores in repo1. There is not a lot of value in comparing restores in other repos and it would require a - # lot of changes to the Perl test harness. - if ($iRepoDefault == 1) - { - $self->restoreCompare($strBackupExpected, dclone($rhExpectedManifest), $bTablespace); - } - } -} - -#################################################################################################################################### -# restoreCompare -#################################################################################################################################### -sub restoreCompare -{ - my $self = shift; - my $strBackup = shift; - my $oExpectedManifestRef = shift; - my $bTablespace = shift; - - my $strTestPath = $self->testPath(); - - # Get the backup host - my $oHostGroup = hostGroupGet(); - my $oHostBackup = defined($oHostGroup->hostGet(HOST_BACKUP, true)) ? $oHostGroup->hostGet(HOST_BACKUP) : $self; - - # Load the last manifest if it exists - my $oLastManifest = undef; - - if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR})) - { - my $oExpectedManifest = - new pgBackRestTest::Env::Manifest( - $self->repoBackupPath( - ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST), - {strCipherPass => $oHostBackup->cipherPassManifest()}); - - # Get the --delta option from the backup manifest so the actual manifest can be built the same way for comparison - $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = - $oExpectedManifest->get(MANIFEST_SECTION_BACKUP_OPTION, &MANIFEST_KEY_DELTA); - - $oLastManifest = - new pgBackRestTest::Env::Manifest( - $self->repoBackupPath( - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST), - {strCipherPass => $oHostBackup->cipherPassManifest()}); - } - - # Generate the tablespace map for real backups - my $oTablespaceMap = undef; - - if (!$self->synthetic()) - { - # Tablespace_map file is not restored in versions >= 9.5 because it interferes with internal remapping features. - if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} >= PG_VERSION_95) - { - delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/tablespace_map'}); - } - - foreach my $strTarget (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}})) - { - if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $iTablespaceId = - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID}; - - $$oTablespaceMap{$iTablespaceId} = - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME}; - } - } - } - - # Generate the actual manifest - my $strDbClusterPath = - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH}; - - if (defined($bTablespace) && !$bTablespace) - { - foreach my $strTarget (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}})) - { - if ($$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} eq - MANIFEST_VALUE_LINK && - defined($$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $strRemapPath; - my $iTablespaceName = - $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME}; - - $strRemapPath = "../../tablespace/${iTablespaceName}"; - - $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; - $$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK}{MANIFEST_TARGET_PGDATA . "/${strTarget}"} - {&MANIFEST_SUBKEY_DESTINATION} = $strRemapPath; - } - } - } - - my $oActualManifest = new pgBackRestTest::Env::Manifest( - "${strTestPath}/" . FILE_MANIFEST, - {bLoad => false, strDbVersion => $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}, - iDbCatalogVersion => $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}, - oStorage => storageTest()}); - - # Build the actual manifest using the delta setting that was actually used by the latest backup if one exists - $oActualManifest->build(storageTest(), $strDbClusterPath, $oLastManifest, false, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}, $oTablespaceMap); - $oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_TYPE, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}); - - my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); - - foreach my $strName ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # Remove repo checksum - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{'rck'}); - - # When bundling zero-length files will not have a reference - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'} && - $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_SIZE} == 0) - { - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); - } - - # If synthetic match checksum errors since they can't be verified here - if ($self->synthetic) - { - my $bChecksumPage = $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}; - - if (defined($bChecksumPage)) - { - $oActualManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, $bChecksumPage); - - if (!$bChecksumPage && - defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR})) - { - $oActualManifest->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, - $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); - } - } - } - # Else if page checksums are enabled make sure the correct files are being checksummed - else - { - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}) - { - if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}) != - isChecksumPage($strName)) - { - confess - "check-page actual for ${strName} is " . - ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, - MANIFEST_SUBKEY_CHECKSUM_PAGE) ? 'set' : '[undef]') . - ' but isChecksumPage() says it should be ' . - (isChecksumPage($strName) ? 'set' : '[undef]') . '.'; - } - - # Because the page checksum flag is copied to incr and diff from the previous backup but further processing is not - # done, they can't be expected to match so delete them. - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE); - } - } - - if (!$self->synthetic()) - { - $oActualManifest->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{size}); - } - - # Remove repo-size, bn*, bi* from the manifest - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bni"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bni"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bno"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bno"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bi"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bi"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bic"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bic"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bim"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bim"}); - - if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0) - { - my $oStat = lstat($oActualManifest->dbPathGet($strSectionPath, $strName)); - - # When performing a selective restore, the files for the database(s) that are not restored are still copied but as empty - # sparse files (blocks == 0). If the file is not a sparse file or is a link, then get the actual checksum for comparison - if ($oStat->blocks > 0 || S_ISLNK($oStat->mode)) - { - my ($strHash) = storageTest()->hashSize($oActualManifest->dbPathGet($strSectionPath, $strName)); - - $oActualManifest->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM, $strHash); - - # If the delta option was set, it is possible that the checksum on the file changed from the last manifest. If so, - # then the file was expected to be copied by the backup and therefore the reference would have been removed. - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}) - { - # If the actual checksum and last manifest checksum don't match, remove the reference - if (defined($oLastManifest) && - $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM) && - $strHash ne $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)) - { - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); - } - } - } - else - { - # If there is a sparse file, remove the checksum and reference since they may or may not match. In this case, it is - # not important to check them since it is known that the file was intentionally not restored. - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM); - delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM}); - - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); - delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REFERENCE}); - } - } - } - - # If PostgreSQL >= 12 don't compare postgresql.auto.conf since it will have recovery settings written into it - if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} >= PG_VERSION_12) - { - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/postgresql.auto.conf'}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, 'pg_data/postgresql.auto.conf'); - } - - # If the link section is empty then delete it and the default section - if (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_LINK}}) == 0) - { - delete($$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK}); - delete($$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK . ':default'}); - } - - # Set actual to expected for settings that always change from backup to backup - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_CHECK}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_COPY}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BUFFER_SIZE, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_LEVEL, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_LEVEL_NETWORK, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_PROCESS_MAX, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX}); - $oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}); - - $oActualManifest->set(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CONTROL, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CONTROL}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_SYSTEM_ID, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_SYSTEM_ID}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_ID, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_ID}); - - $oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, - ${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_VERSION}); - - # Copy passphrase if one exists - if (defined($oExpectedManifestRef->{&INI_SECTION_CIPHER}) && - defined($oExpectedManifestRef->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS})) - { - $oActualManifest->set(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS, undef, - $oExpectedManifestRef->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}); - } - - # This option won't be set in the actual manifest - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}); - - if ($self->synthetic()) - { - $oActualManifest->remove(MANIFEST_SECTION_BACKUP); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}); - } - else - { - $oActualManifest->set( - INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef, $oExpectedManifestRef->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, 'backup-reference', undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-reference'}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE}); - - if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'})) - { - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, 'backup-bundle', undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'}); - } - - if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle-raw'})) - { - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, 'backup-bundle-raw', undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle-raw'}); - } - - # Delete block incr headers since old Perl manifest code will not generate them - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-block-incr'}); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-block-incr-size'}); - - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_START, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_START}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_STOP, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_STOP}); - - if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_START})) - { - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_START}); - } - - if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_STOP}) - { - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_STOP}); - } - } - - # Check that archive status exists in the manifest for an online backup - my $strArchiveStatusPath = MANIFEST_TARGET_PGDATA . qw{/} . $oActualManifest->walPath() . qw{/} . DB_PATH_ARCHIVESTATUS; - - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE} && - !defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_PATH}{$strArchiveStatusPath})) - { - confess &log(ERROR, "${strArchiveStatusPath} expected for online backup", ERROR_ASSERT); - } - - # Delete the list of DBs - delete($$oExpectedManifestRef{&MANIFEST_SECTION_DB}); - - # Only update defaults if the expect manifest is synthetic. If loaded from a file the defaults will already be correct. - if ($self->synthetic()) - { - $self->manifestDefault($oExpectedManifestRef); - } - - # Newer Perls will change this variable to a number whenever a numeric comparison is performed. It is expected to be a string so - # make sure it is one before saving. - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} .= ''; - - storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent})); - storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifestRef)); - - executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest"); - - storageTest()->remove("${strTestPath}/expected.manifest"); - storageTest()->remove("${strTestPath}/actual.manifest"); -} - -#################################################################################################################################### -# Get repo backup/archive path -#################################################################################################################################### -sub repoSubPath -{ - my $self = shift; - my $strSubPath = shift; - my $strPath = shift; - my $iRepo = shift; - - my $strRepoPath = $self->repoPath(); - - if (defined($iRepo) && $iRepo == 2) - { - $strRepoPath = $self->repo2Path(); - } - - return - ($strRepoPath eq '/' ? '' : $strRepoPath) . "/${strSubPath}/" . $self->stanza() . - (defined($strPath) ? "/${strPath}" : ''); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub backrestConfig {return shift->{strBackRestConfig}} -sub backupDestination {return shift->{strBackupDestination}} -sub backrestExe {return testRunGet()->backrestExe()} -sub bogusHost {return shift->{bBogusHost}} -sub hardLink {return shift->{bHardLink}} -sub hasLink {storageRepo()->capability(STORAGE_CAPABILITY_LINK)} -sub isFS {storageRepo()->type() ne STORAGE_OBJECT} -sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()} -sub isHostDbPrimary {return shift->nameGet() eq HOST_DB_PRIMARY} -sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY} -sub isHostDb {my $self = shift; return $self->isHostDbPrimary() || $self->isHostDbStandby()} -sub lockPath {return shift->{strLockPath}} -sub logPath {return shift->{strLogPath}} -sub repoArchivePath {return shift->repoSubPath('archive', shift)} -sub repoBackupPath {return shift->repoSubPath('backup', shift, shift)} -sub repoPath {return shift->{strRepoPath}} -sub repo2Path {return shift->{strRepo2Path}} -sub repoEncrypt {return shift->{bRepoEncrypt}} -sub stanza {return testRunGet()->stanza()} -sub synthetic {return shift->{bSynthetic}} -sub cipherPassManifest {return shift->{strCipherPassManifest}} -sub cipherPassArchive {return shift->{strCipherPassArchive}} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm deleted file mode 100644 index 229055bbe..000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm +++ /dev/null @@ -1,139 +0,0 @@ -#################################################################################################################################### -# HostBackupTest.pm - Backup host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostBaseTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::JobTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::VmTest; - -#################################################################################################################################### -# Host constants -#################################################################################################################################### -use constant HOST_BASE => 'base'; - push @EXPORT, qw(HOST_BASE); -use constant HOST_DB_PRIMARY => 'db-primary'; - push @EXPORT, qw(HOST_DB_PRIMARY); -use constant HOST_DB_STANDBY => 'db-standby'; - push @EXPORT, qw(HOST_DB_STANDBY); -use constant HOST_BACKUP => 'backup'; - push @EXPORT, qw(HOST_BACKUP); -use constant HOST_GCS => 'gcs'; - push @EXPORT, qw(HOST_GCS); -use constant HOST_AZURE => 'azure'; - push @EXPORT, qw(HOST_AZURE); -use constant HOST_S3 => 's3-server'; - push @EXPORT, qw(HOST_S3); -use constant HOST_SFTP => 'sftp-srvr'; - push @EXPORT, qw(HOST_SFTP); - -#################################################################################################################################### -# CA/cert/key constants -#################################################################################################################################### -use constant HOST_CERT_PATH => '/test/certificate/'; - -use constant HOST_CLIENT_CERT => HOST_CERT_PATH . 'pgbackrest-test-client.crt'; - push @EXPORT, qw(HOST_CLIENT_CERT); -use constant HOST_CLIENT_KEY => HOST_CERT_PATH . 'pgbackrest-test-client.key'; - push @EXPORT, qw(HOST_CLIENT_KEY); - -use constant HOST_SERVER_CA => HOST_CERT_PATH . 'pgbackrest-test-ca.crt'; - push @EXPORT, qw(HOST_SERVER_CA); -use constant HOST_SERVER_CERT => HOST_CERT_PATH . 'pgbackrest-test-server.crt'; -use constant HOST_SERVER_KEY => HOST_CERT_PATH . 'pgbackrest-test-server.key'; - -#################################################################################################################################### -# SFTP key constants -#################################################################################################################################### -use constant SSH_KEY_PATH => '/test/certificate/ssh/'; - -use constant SSH_PRIVATE_KEY => SSH_KEY_PATH . 'id_rsa'; - push @EXPORT, qw(SSH_PRIVATE_KEY); -use constant SSH_PUBLIC_KEY => SSH_KEY_PATH . 'id_rsa.pub'; - push @EXPORT, qw(SSH_PUBLIC_KEY); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strName, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strName', default => HOST_BASE, trace => true}, - {name => 'oParam', required => false, trace => true}, - ); - - my $strTestPath = testRunGet()->testPath() . ($strName eq HOST_BASE ? '' : "/${strName}"); - storageTest()->pathCreate($strTestPath, {strMode => '0770'}); - - # Make sure keys have the correct permissions - if (chmod(0600, testRunGet()->basePath() . HOST_SERVER_KEY, testRunGet()->basePath() . HOST_CLIENT_KEY) != 2) - { - confess "unable to set mode on keys"; - } - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strBinPath = dirname(dirname($strTestPath)) . '/bin/' . testRunGet()->vm() . '/' . PROJECT_EXE; - my $strContainer = 'test-' . testRunGet()->vmId() . "-$strName"; - - my $self = $class->SUPER::new( - $strName, $strContainer, $$oParam{strImage}, $$oParam{strUser}, - ["${strProjectPath}:${strProjectPath}", "${strTestPath}:${strTestPath}", "${strBinPath}:${strBinPath}:ro"], undef, - $oParam->{bTls} ? - 'server --log-level-console=debug --tls-server-ca-file=' . testRunGet()->basePath() . HOST_SERVER_CA . - ' --tls-server-cert-file=' . testRunGet()->basePath() . HOST_SERVER_CERT . ' --tls-server-key-file=' . - testRunGet()->basePath() . HOST_SERVER_KEY . ' --tls-server-auth=pgbackrest-client=* --tls-server-address=0.0.0.0' : - undef, - undef, $oParam->{bTls} ? testRunGet()->backrestExe() : undef); - bless $self, $class; - - # Set test path - $self->{strTestPath} = $strTestPath; - - # Set permissions on the test path - $self->executeSimple('chown -R ' . $self->userGet() . ':'. TEST_GROUP . ' ' . $self->testPath(), undef, 'root'); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub testPath {return shift->{strTestPath}} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm deleted file mode 100644 index 9418f35ce..000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm +++ /dev/null @@ -1,217 +0,0 @@ -#################################################################################################################################### -# HostDbTest.pm - Database host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostDbCommonTest; -use parent 'pgBackRestTest::Env::Host::HostBackupTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Test WAL size -#################################################################################################################################### -use constant PG_WAL_SIZE_TEST => 16777216; - -#################################################################################################################################### -# Host defaults -#################################################################################################################################### -use constant HOST_PATH_SPOOL => 'spool'; -use constant HOST_PATH_DB => 'db'; -use constant HOST_PATH_DB_BASE => 'base'; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - # Get host group - my $oHostGroup = hostGroupGet(); - - # Is standby? - my $bStandby = defined($$oParam{bStandby}) && $$oParam{bStandby} ? true : false; - - my $self = $class->SUPER::new( - { - strName => $bStandby ? HOST_DB_STANDBY : HOST_DB_PRIMARY, - strImage => $$oParam{strImage}, - bTls => $oParam->{bTls}, - strBackupDestination => $$oParam{strBackupDestination}, - bSynthetic => $$oParam{bSynthetic}, - bRepoLocal => $oParam->{bRepoLocal}, - bRepoEncrypt => $oParam->{bRepoEncrypt}, - }); - bless $self, $class; - - # Set parameters - $self->{bStandby} = $bStandby; - - $self->{strDbPath} = $self->testPath() . '/' . HOST_PATH_DB; - $self->{strDbBasePath} = $self->dbPath() . '/' . HOST_PATH_DB_BASE; - $self->{strTablespacePath} = $self->dbPath() . '/tablespace'; - - storageTest()->pathCreate($self->dbBasePath(), {strMode => '0700', bCreateParent => true}); - - $self->{strSpoolPath} = $self->testPath() . '/' . HOST_PATH_SPOOL; - storageTest()->pathCreate($self->spoolPath()); - - # Initialize linkRemap Hashes - $self->{hLinkRemap} = {}; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# archivePush -#################################################################################################################################### -sub archivePush -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strWalPath, - $strArchiveTestFile, - $iArchiveNo, - $iExpectedError, - $bAsync, - $strOptionalParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->archivePush', \@_, - {name => 'strWalPath'}, - {name => 'strArchiveTestFile', required => false}, - {name => 'iArchiveNo', required => false}, - {name => 'iExpectedError', required => false}, - {name => 'bAsync', default => true}, - {name => 'strOptionalParam', required => false}, - ); - - my $strSourceFile; - - if (defined($strArchiveTestFile)) - { - $strSourceFile = "${strWalPath}/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)); - - storageTest()->copy($strArchiveTestFile, storageTest()->openWrite($strSourceFile, {bPathCreate => true})); - - storageTest()->pathCreate("${strWalPath}/archive_status/", {bIgnoreExists => true, bCreateParent => true}); - storageTest()->put("${strWalPath}/archive_status/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)) . '.ready'); - } - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --log-level-console=warn --archive-push-queue-max=' . int(2 * PG_WAL_SIZE_TEST) . - ' --stanza=' . $self->stanza() . - ($bAsync ? '' : ' --no-archive-async') . - " archive-push" . (defined($strSourceFile) ? " ${strSourceFile}" : '') . - (defined($strOptionalParam) ? " ${strOptionalParam}" : ''), - {iExpectedExitStatus => $iExpectedError, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# linkRemap -#################################################################################################################################### -sub linkRemap -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strTarget, - $strDestination - ) = - logDebugParam - ( - __PACKAGE__ . '->linkRemap', \@_, - {name => 'strTarget'}, - {name => 'strDestination'}, - ); - - ${$self->{hLinkRemap}}{$strTarget} = $strDestination; - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub dbPath {return shift->{strDbPath};} - -sub dbBasePath -{ - my $self = shift; - my $iIndex = shift; - - return $self->{strDbBasePath} . (defined($iIndex) ? "-${iIndex}" : ''); -} - -sub spoolPath {return shift->{strSpoolPath}} -sub standby {return shift->{bStandby}} - -sub tablespacePath -{ - my $self = shift; - my $iTablespace = shift; - my $iIndex = shift; - - return - $self->{strTablespacePath} . - (defined($iTablespace) ? "/ts${iTablespace}" . - (defined($iIndex) ? "-${iIndex}" : '') : ''); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm deleted file mode 100644 index ed6af2de3..000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm +++ /dev/null @@ -1,717 +0,0 @@ -#################################################################################################################################### -# HostDbTest.pm - Database host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostDbSyntheticTest; -use parent 'pgBackRestTest::Env::Host::HostDbCommonTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use Fcntl ':mode'; -use File::Basename qw(basename dirname); -use File::stat; - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - my $self = $class->SUPER::new( - { - strImage => containerRepo() . ':' . testRunGet()->vm() . "-test", - bTls => $oParam->{bTls}, - strBackupDestination => $$oParam{strBackupDestination}, - bSynthetic => true, - bStandby => $$oParam{bStandby}, - bRepoLocal => $oParam->{bRepoLocal}, - bRepoEncrypt => $oParam->{bRepoEncrypt}, - }); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# dbFileCreate -# -# Create a file specifying content, mode, and time. -#################################################################################################################################### -sub dbFileCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $strContent = shift; - my $lTime = shift; - my $strMode = shift; - - # Check that strTarget is a valid - my $strPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH}; - - if (!defined($strPath)) - { - confess &log(ERROR, "${strTarget} not a valid target: \n" . Dumper(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET})); - } - - # Get tablespace path if this is a tablespace - my $strPgPath; - - if (index($strTarget, DB_PATH_PGTBLSPC . '/') == 0) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strPgPath = 'PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - # Create actual file location - my $strPathFile = $strPath . - (defined($strPgPath) ? "/${strPgPath}" : '') . "/${strFile}"; - - if (index($strPathFile, '/') != 0) - { - $strPathFile = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} . '/' . - (defined(dirname($strPathFile)) ? dirname($strPathFile) : '') . "/${strPathFile}"; - } - - # Create the file - testFileCreate($strPathFile, $strContent, $lTime, $strMode); - - # Return path to created file - return $strPathFile; -} - -#################################################################################################################################### -# dbFileRemove -#################################################################################################################################### -sub dbFileRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $bIgnoreMissing = shift; - - # Get actual path location - my $strDbFile = $self->manifestDbPathGet($oManifestRef, $strTarget, $strFile); - - # Remove the file - if (!(defined($bIgnoreMissing) && $bIgnoreMissing && !(-e $strDbFile))) - { - testFileRemove($strDbFile); - } - - return $strDbFile; -} - -#################################################################################################################################### -# dbLinkCreate -# -# Create a file specifying content, mode, and time. -#################################################################################################################################### -sub dbLinkCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $strDestination = shift; - - # Create actual file location - my $strDbFile = $self->manifestDbPathGet($oManifestRef, $strTarget, $strFile); - - # Create the file - testLinkCreate($strDbFile, $strDestination); - - # Return path to created file - return $strDbFile; -} - -#################################################################################################################################### -# manifestDbPathGet -# -# Get the db path based on the target and file passed. -#################################################################################################################################### -sub manifestDbPathGet -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - - # Determine the manifest key - my $strDbPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH}; - - # If target is a tablespace - if (defined(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strDbPath .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - $strDbPath .= defined($strFile) ? "/${strFile}" : ''; - - return $strDbPath; -} - -#################################################################################################################################### -# manifestFileCreate -# -# Create a file specifying content, mode, and time and add it to the manifest. -#################################################################################################################################### -sub manifestFileCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $strContent = shift; - my $strChecksum = shift; - my $lTime = shift; - my $strMode = shift; - my $bPrimary = shift; - my $strChecksumPageError = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strTarget, $strFile); - - # Create the file - my $strPathFile = $self->dbFileCreate($oManifestRef, $strTarget, $strFile, $strContent, $lTime, $strMode); - - # Stat the file - my $oStat = lstat($strPathFile); - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = - sprintf('%04o', S_IMODE($oStat->mode)); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime; - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size; - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_REFERENCE}); - - my $bChecksumPage = defined($strChecksumPageError) ? false : (isChecksumPage($strManifestKey) ? true : undef); - - if (defined($bChecksumPage)) - { - $oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE} = - $bChecksumPage ? JSON::PP::true : JSON::PP::false; - - if (!$bChecksumPage && $strChecksumPageError ne '0') - { - my @iyChecksumPageError = eval($strChecksumPageError); - - $oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR} = - \@iyChecksumPageError; - } - else - { - delete($oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); - } - } - - if (defined($strChecksum)) - { - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{checksum} = $strChecksum; - } -} - -#################################################################################################################################### -# manifestFileRemove -# -# Remove a file from disk and (optionally) the manifest. -#################################################################################################################################### -sub manifestFileRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strTarget, $strFile); - - # Remove the file - $self->dbFileRemove($oManifestRef, $strTarget, $strFile, true); - - # Remove from manifest - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}); -} - -#################################################################################################################################### -# manifestKeyGet -# -# Get the manifest key based on the target and file/path/link passed. -#################################################################################################################################### -sub manifestKeyGet -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - - # Determine the manifest key - my $strManifestKey = $strTarget; - - # If target is a tablespace - if (defined(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strManifestKey .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - $strManifestKey .= (defined($strFile) ? "/$strFile" : ''); - - return $strManifestKey; -} - -#################################################################################################################################### -# manifestLinkCreate -# -# Create a link and add it to the manifest. -#################################################################################################################################### -sub manifestLinkCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strPath = shift; - my $strFile = shift; - my $strDestination = shift; - my $bPrimary = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strFile); - - # Load target - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH} = $strDestination; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_LINK; - - # Create the link - my $strDbFile = $self->dbLinkCreate($oManifestRef, $strPath, $strFile, $strDestination); - - # Stat the link - my $oStat = lstat($strDbFile); - - # Check for errors in stat - if (!defined($oStat)) - { - confess 'unable to stat ${strDbFile}'; - } - - # Load file into manifest - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_DESTINATION} = $strDestination; - - # Stat what the link is pointing to - my $strDestinationFile = $strDestination; - - if (index($strDestinationFile, '/') != 0) - { - $strDestinationFile = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} . '/' . - (defined(dirname($strPath)) ? dirname($strPath) : '') . "/${strDestination}"; - } - - $oStat = lstat($strDestinationFile); - - my $strSection = MANIFEST_SECTION_TARGET_PATH; - - if (S_ISREG($oStat->mode)) - { - $strSection = MANIFEST_SECTION_TARGET_FILE; - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size; - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime; - (${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM}) = storageTest()->hashSize($strDestinationFile); - - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_FILE} = - basename(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH}); - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH} = - dirname(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH}); - } - # Allow a link to a link to be created to test that backrest errors out correctly - elsif (S_ISLNK($oStat->mode)) - { - $strSection = MANIFEST_SECTION_TARGET_LINK; - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_DESTINATION} = $strDestination; - } - elsif (!S_ISDIR($oStat->mode)) - { - confess &log(ASSERT, "unrecognized file type for file $strDestinationFile"); - } - - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = sprintf('%04o', S_IMODE($oStat->mode)); -} - -#################################################################################################################################### -# manifestLinkMap -# -# Remap links to new directories/files -#################################################################################################################################### -sub manifestLinkMap -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strDestination = shift; - - if ($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK) - { - confess "cannot map target ${strTarget} because it is not a link"; - } - - if (defined($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - confess "tablespace ${strTarget} cannot be remapped with this function"; - } - - if (defined($strDestination)) - { - confess "GENERAL LINK REMAP NOT IMPLEMENTED"; - } - else - { - delete($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}); - delete($$oManifestRef{&MANIFEST_SECTION_TARGET_LINK}{$strTarget}); - } -} - -#################################################################################################################################### -# manifestLinkRemove -# -# Create a link and add it to the manifest. -#################################################################################################################################### -sub manifestLinkRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strPath = shift; - my $strFile = shift; - - # Delete the link - my $strDbFile = $self->dbFileRemove($oManifestRef, $strPath, $strFile); - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strFile); - - # Delete from manifest - delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strManifestKey}); -} - -#################################################################################################################################### -# manifestPathCreate -# -# Create a path specifying mode and add it to the manifest. -#################################################################################################################################### -sub manifestPathCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strPath = shift; - my $strSubPath = shift; - my $strMode = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strSubPath); - - # Create the db path - my $strDbPath = $self->dbPathCreate($oManifestRef, $strPath, $strSubPath, defined($strMode) ? $strMode : '0700'); - - # Stat the file - my $oStat = lstat($strDbPath); - - # Check for errors in stat - if (!defined($oStat)) - { - confess 'unable to stat ${strSubPath}'; - } - - # Load file into manifest - my $strSection = MANIFEST_SECTION_TARGET_PATH; - - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = sprintf('%04o', S_IMODE($oStat->mode)); -} - -#################################################################################################################################### -# manifestReference -# -# Update all files that do not have a reference with the supplied reference. -#################################################################################################################################### -sub manifestReference -{ - my $self = shift; - my $oManifestRef = shift; - my $strReference = shift; - my $bClear = shift; - - # Set prior backup - if (defined($strReference)) - { - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} = $strReference; - } - else - { - delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR}); - } - - # Find all file sections - foreach my $strSectionFile (sort(keys(%$oManifestRef))) - { - # Skip non-file sections - if ($strSectionFile !~ /\:file$/) - { - next; - } - - foreach my $strFile (sort(keys(%{${$oManifestRef}{$strSectionFile}}))) - { - if (!defined($strReference)) - { - delete(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}); - } - elsif (defined($bClear) && $bClear) - { - if (defined(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}) && - ${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE} ne $strReference) - { - delete(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}); - } - } - elsif (!defined(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE})) - { - ${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE} = $strReference; - } - } - } -} - -#################################################################################################################################### -# manifestTablespaceCreate -# -# Create a tablespace specifying mode and add it to the manifest. -#################################################################################################################################### -sub manifestTablespaceCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $iOid = shift; - my $strMode = shift; - - # Load linked path into manifest - my $strLinkPath = $self->tablespacePath($iOid); - my $strTarget = MANIFEST_TARGET_PGTBLSPC . "/${iOid}"; - my $oStat = lstat($strLinkPath); - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_MODE} = - sprintf('%04o', S_IMODE($oStat->mode)); - - # Create the tablespace path if it does not exist - my $strTablespacePath = $strLinkPath; - my $strPathTarget = $strTarget; - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - my $strTablespaceId = 'PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - - $strTablespacePath .= "/${strTablespaceId}"; - $strPathTarget .= "/${strTablespaceId}"; - - if (!-e $strTablespacePath) - { - storageTest()->pathCreate($strTablespacePath, {strMode => defined($strMode) ? $strMode : '0700'}); - } - - # Load tablespace path into manifest - $oStat = lstat($strTablespacePath); - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGTBLSPC} = - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}; - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_MODE} = - sprintf('%04o', S_IMODE($oStat->mode)); - - # Create the link in DB_PATH_PGTBLSPC - my $strLink = $self->dbBasePath() . '/' . DB_PATH_PGTBLSPC . "/${iOid}"; - - symlink($strLinkPath, $strLink) - or confess "unable to link ${strLink} to ${strLinkPath}"; - - # Load link into the manifest - $oStat = lstat($strLink); - my $strLinkTarget = MANIFEST_TARGET_PGDATA . "/${strTarget}"; - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_DESTINATION} = $strLinkPath; - - # Load tablespace target into the manifest - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH} = $strLinkPath; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_LINK; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID} = $iOid; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME} = "ts${iOid}"; -} - -#################################################################################################################################### -# manifestTablespaceDrop -# -# Drop a tablespace add remove it from the manifest. -#################################################################################################################################### -sub manifestTablespaceDrop -{ - my $self = shift; - my $oManifestRef = shift; - my $iOid = shift; - my $iIndex = shift; - - # Remove tablespace path/file/link from manifest - my $strTarget = DB_PATH_PGTBLSPC . "/${iOid}"; - - # Remove manifest path, link, target - delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{&MANIFEST_TARGET_PGDATA . "/${strTarget}"}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}); - - # Remove nested manifest files and paths - foreach my $strSection (&MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_FILE) - { - foreach my $strFile (keys(%{${$oManifestRef}{$strSection}})) - { - if (index($strFile, "${strTarget}/") == 0) - { - delete($$oManifestRef{$strSection}{$strFile}); - } - } - } - - # Drop the link in DB_PATH_PGTBLSPC - testFileRemove($self->dbBasePath($iIndex) . "/${strTarget}"); -} - -#################################################################################################################################### -# dbPathCreate -# -# Create a path specifying mode. -#################################################################################################################################### -sub dbPathCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strSubPath = shift; - my $strMode = shift; - - # Create final file location - my $strFinalPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH}; - - # Get tablespace path if this is a tablespace - if (index($strTarget, DB_PATH_PGTBLSPC . '/') == 0) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strFinalPath .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - $strFinalPath .= (defined($strSubPath) ? "/${strSubPath}" : ''); - - # Create the path - if (!(-e $strFinalPath)) - { - storageTest()->pathCreate($strFinalPath, {strMode => $strMode}); - } - - return $strFinalPath; -} - -#################################################################################################################################### -# dbPathMode -# -# Change the mode of a path. -#################################################################################################################################### -sub dbPathMode -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strPath = shift; - my $strMode = shift; - - # Get the db path - my $strDbPath = $self->manifestDbPathGet($oManifestRef, $strTarget, $strPath); - - testPathMode($strDbPath, $strMode); - - return $strDbPath; -} - -#################################################################################################################################### -# dbPathRemove -# -# Remove a path. -#################################################################################################################################### -sub dbPathRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strPath = shift; - - # Get the db path - my $strDbPath = $self->manifestDbPathGet($oManifestRef, $strTarget, $strPath); - - # Create the path - testPathRemove($strDbPath); - - return $strDbPath; -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm deleted file mode 100644 index 10a292d4f..000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm +++ /dev/null @@ -1,528 +0,0 @@ -#################################################################################################################################### -# HostDbTest.pm - Database host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostDbTest; -use parent 'pgBackRestTest::Env::Host::HostDbCommonTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use DBI; -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(basename); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Db defaults -#################################################################################################################################### -use constant HOST_DB_DEFAULT => 'postgres'; -use constant HOST_DB_TIMEOUT => 30; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - # Get db version - my $strDbVersion = testRunGet()->pgVersion(); - - my $self = $class->SUPER::new( - { - strImage => containerRepo() . ':' . testRunGet()->vm() . "-test", - bTls => $oParam->{bTls}, - strBackupDestination => $$oParam{strBackupDestination}, - bStandby => $$oParam{bStandby}, - bRepoLocal => $oParam->{bRepoLocal}, - bRepoEncrypt => $oParam->{bRepoEncrypt}, - }); - bless $self, $class; - - # Set parameters - $self->{strPgSocketPath} = $self->dbPath(); - $self->{iPgPort} = defined($$oParam{bStandby}) && $$oParam{bStandby} ? 6544 : 6543; - - $self->{strPgLogPath} = $self->testPath(); - $self->{strPgLogFile} = $self->pgLogPath() . '/postgresql.log'; - - # Get Db version - if (defined($strDbVersion)) - { - my $strOutLog = $self->executeSimple($self->pgBinPath() . '/postgres --version'); - - my @stryVersionToken = split(/ /, $strOutLog); - @stryVersionToken = split(/\./, $stryVersionToken[2]); - my $strDbVersionActual = - trim($stryVersionToken[0]) . - (defined($stryVersionToken[1]) && trim($stryVersionToken[0]) < 10 ? '.' . trim($stryVersionToken[1]) : ''); - - # Warn if this is a devel/alpha/beta version - my $strVersionRegExp = '(devel|((alpha|beta|rc)[0-9]+))$'; - - if ($strDbVersionActual =~ /$strVersionRegExp/) - { - my $strDevVersion = $strDbVersionActual; - $strDbVersionActual =~ s/$strVersionRegExp//; - $strDevVersion = substr($strDevVersion, length($strDbVersionActual)); - - if (!defined($$oParam{bStandby}) || !$$oParam{bStandby}) - { - &log(WARN, 'Testing against ' . trim($strOutLog) . " ${strDevVersion}"); - } - } - elsif (!defined($$oParam{bStandby}) || !$$oParam{bStandby}) - { - &log(INFO, 'Testing against ' . trim($strOutLog)); - } - - # Don't run unit tests for unsupported versions - my @stryVersionSupport = versionSupport(); - - if ($strDbVersionActual < $stryVersionSupport[0]) - { - confess &log(ERROR, "only PostgreSQL version $stryVersionSupport[0] and up are supported"); - } - - if ($strDbVersion ne $strDbVersionActual) - { - confess &log(ERROR, "actual database version ${strDbVersionActual} does not match expected version ${strDbVersion}"); - } - } - - # Create wal directory - storageTest()->pathCreate($self->dbPath() . '/pg_' . $self->walId(), {strMode => '0700'}); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# sqlConnect -#################################################################################################################################### -sub sqlConnect -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $iTimeout = defined($$hParam{iTimeout}) ? $$hParam{iTimeout} : HOST_DB_TIMEOUT; - my $strDb = defined($$hParam{strDb}) ? $$hParam{strDb} : HOST_DB_DEFAULT; - - # If not connected - if (!defined($self->{db}{$strDb}{hDb})) - { - # Retry until connection is successful - my $oWait = waitInit($iTimeout); - - do - { - # Connect to the db (whether it is local or remote) - $self->{db}{$strDb}{hDb} = - DBI->connect( - "dbi:Pg:dbname=${strDb};port=" . $self->pgPort() . ';host=' . $self->pgSocketPath(), - $self->userGet(), undef, - {AutoCommit => 0, RaiseError => 0, PrintError => 0}); - - return $self->{db}{$strDb}{hDb} if $self->{db}{$strDb}{hDb}; - } - while (!defined($self->{db}{$strDb}{hDb}) && waitMore($oWait)); - - # Error if unable to connect - if (!defined($self->{db}{$strDb}{hDb})) - { - confess &log(ERROR, "unable to connect to PostgreSQL after ${iTimeout} second(s):\n" . $DBI::errstr, ERROR_DB_CONNECT); - } - } - - return $self->{db}{$strDb}{hDb}; -} - -#################################################################################################################################### -# sqlDisconnect -#################################################################################################################################### -sub sqlDisconnect -{ - my $self = shift; - my $hParam = shift; - - foreach my $strDb (keys(%{$self->{db}})) - { - if (defined($$hParam{$strDb}) && $$hParam{$strDb} ne $strDb) - { - next; - } - - if (defined($self->{db}{$strDb}{hDb})) - { - $self->{db}{$strDb}{hDb}->disconnect(); - undef($self->{db}{$strDb}{hDb}); - } - } -} - -#################################################################################################################################### -# sqlExecute -#################################################################################################################################### -sub sqlExecute -{ - my $self = shift; - my $strSql = shift; - my $hParam = shift; - - # Set defaults - my $bCheckPoint = defined($$hParam{bCheckPoint}) ? $$hParam{bCheckPoint} : false; - my $bCommit = defined($$hParam{bCommit}) ? $$hParam{bCommit} : true; - - # Get the db handle - my $hDb = $self->sqlConnect({strDb => $$hParam{strDb}}); - - # Set autocommit on/off - $hDb->{AutoCommit} = defined($$hParam{bAutoCommit}) ? ($$hParam{bAutoCommit} ? true : false) : false; - - # Log and execute the statement - &log(DETAIL, "SQL: ${strSql}"); - - my $hStatement = $hDb->prepare($strSql); - - $hStatement->execute() or - confess &log(ERROR, "Unable to execute: ${strSql}\n" . $DBI::errstr); - $hStatement->finish(); - - if ($bCommit && !$hDb->{AutoCommit}) - { - $self->sqlCommit(); - } - - # Perform a checkpoint if requested - if ($bCheckPoint) - { - $self->sqlExecute('checkpoint', {bCommit => false, bCheckPoint => false}); - } - - # Set autocommit off - $hDb->{AutoCommit} = 0; -} - -#################################################################################################################################### -# sqlSelect -#################################################################################################################################### -sub sqlSelect -{ - my $self = shift; - my $strSql = shift; - my $hParam = shift; - - # Get the db handle - my $hDb = $self->sqlConnect({strDb => $$hParam{strDb}}); - - # Log and execute the statement - &log(DEBUG, (defined($$hParam{strDb}) ? "DB: $$hParam{strDb}, " : "") . "SQL: ${strSql}"); - my $hStatement = $hDb->prepare($strSql); - - $hStatement = $hDb->prepare($strSql); - - $hStatement->execute() or - confess &log(ERROR, "Unable to execute: ${strSql}\n" . $DBI::errstr); - - my @oyRow = $hStatement->fetchrow_array(); - - $hStatement->finish(); - - return @oyRow; -} - -#################################################################################################################################### -# sqlSelectOne -#################################################################################################################################### -sub sqlSelectOne -{ - my $self = shift; - my $strSql = shift; - my $hParam = shift; - - return ($self->sqlSelect($strSql, $hParam))[0]; -} - -#################################################################################################################################### -# sqlSelectOneTest -#################################################################################################################################### -sub sqlSelectOneTest -{ - my $self = shift; - my $strSql = shift; - my $strExpectedValue = shift; - my $hParam = shift; - - # Set defaults - my $iTimeout = defined($$hParam{iTimeout}) ? $$hParam{iTimeout} : HOST_DB_TIMEOUT; - - my $lStartTime = time(); - my $strActualValue; - - do - { - $self->sqlConnect($hParam); - $strActualValue = $self->sqlSelectOne($strSql, $hParam); - - if (defined($strActualValue) && $strActualValue eq $strExpectedValue) - { - return; - } - - $self->sqlDisconnect(); - } - while (defined($iTimeout) && (time() - $lStartTime) <= $iTimeout); - - confess &log( - ERROR, "expected value '${strExpectedValue}' from '${strSql}' but actual was '" . - (defined($strActualValue) ? $strActualValue : '[undef]') . "'"); -} - -#################################################################################################################################### -# sqlCommit -#################################################################################################################################### -sub sqlCommit -{ - my $self = shift; - my $hParam = shift; - - my $bCheckPoint = defined($$hParam{bCheckPoint}) ? $$hParam{bCheckPoint} : false; - - $self->sqlExecute('commit', {bCommit => false, bCheckPoint => $bCheckPoint}); -} - -#################################################################################################################################### -# sqlWalRotate -#################################################################################################################################### -sub sqlWalRotate -{ - my $self = shift; - - $self->sqlExecute('select pg_switch_' . $self->walId() . '()', {bCommit => false, bCheckPoint => false}); -} - -#################################################################################################################################### -# clusterCreate -# -# Create the PostgreSQL cluster and start it. -#################################################################################################################################### -sub clusterCreate -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $strWalPath = defined($$hParam{strWalPath}) ? $$hParam{strWalPath} : $self->dbPath() . '/pg_' . $self->walId(); - - $self->executeSimple( - $self->pgBinPath() . '/initdb -k' . - ($self->pgVersion() >= PG_VERSION_11 ? ' --wal-segsize=1' : '') . - ' --' . $self->walId() . "dir=${strWalPath}" . ' --pgdata=' . $self->dbBasePath() . ' --auth=trust'); - - if (!$self->standby()) - { - $self->executeSimple( - "echo 'host replication replicator db-standby trust' >> " . $self->dbBasePath() . '/pg_hba.conf'); - } - - $self->clusterStart( - {bHotStandby => $$hParam{bHotStandby}, bArchive => $$hParam{bArchive}, bArchiveAlways => $$hParam{bArchiveAlways}, - bArchiveInvalid => $$hParam{bArchiveInvalid}}); - - if (!$self->standby()) - { - $self->sqlExecute("create user replicator replication", {bCommit =>true}); - } -} - -#################################################################################################################################### -# clusterStart -# -# Start the PostgreSQL cluster with various test options. -#################################################################################################################################### -sub clusterStart -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $bHotStandby = defined($$hParam{bHotStandby}) ? $$hParam{bHotStandby} : false; - my $bArchive = defined($$hParam{bArchive}) ? $$hParam{bArchive} : true; - my $bArchiveAlways = defined($$hParam{bArchiveAlways}) ? $$hParam{bArchiveAlways} : false; - my $bArchiveInvalid = defined($$hParam{bArchiveInvalid}) ? $$hParam{bArchiveInvalid} : false; - my $bArchiveEnabled = defined($$hParam{bArchiveEnabled}) ? $$hParam{bArchiveEnabled} : true; - - # Make sure postgres is not running - if (-e $self->dbBasePath() . '/' . DB_FILE_POSTMTRPID) - { - confess DB_FILE_POSTMTRPID . ' exists'; - } - - # Create the archive command - my $strArchive = - $self->backrestExe() . ' --stanza=' . ($bArchiveInvalid ? 'bogus' : $self->stanza()) . - ' --config=' . $self->backrestConfig() . ' archive-push %p'; - - # Start the cluster - my $strCommand = - $self->pgBinPath() . '/pg_ctl start -o "-c port=' . $self->pgPort() . - ($self->pgVersion() < PG_VERSION_95 ? ' -c checkpoint_segments=1' : ''); - - if ($bArchiveEnabled) - { - if ($self->pgVersion() >= PG_VERSION_95 && $bArchiveAlways) - { - $strCommand .= " -c archive_mode=always"; - } - else - { - $strCommand .= " -c archive_mode=on"; - } - } - else - { - $strCommand .= " -c archive_mode=off"; - } - - if ($bArchive) - { - $strCommand .= " -c archive_command='${strArchive}'"; - } - else - { - $strCommand .= " -c archive_command=true"; - } - - $strCommand .= ' -c wal_level=hot_standby -c hot_standby=' . ($bHotStandby ? 'on' : 'off'); - - # Force parallel mode on to make sure we are disabling it and there are no issues. This is important for testing that 9.6 - # works since pg_stop_backup() is marked parallel safe and will error if run in a worker. - if ($self->pgVersion() >= PG_VERSION_96) - { - if ($self->pgVersion() >= PG_VERSION_16) - { - $strCommand .= " -c debug_parallel_query='on'"; - } - else - { - $strCommand .= " -c force_parallel_mode='on'"; - } - - $strCommand .= " -c max_parallel_workers_per_gather=2"; - } - - $strCommand .= - ' -c max_wal_senders=3' . - ' -c listen_addresses=\'*\'' . - ' -c log_directory=\'' . $self->pgLogPath() . "'" . - ' -c log_filename=\'' . basename($self->pgLogFile()) . "'" . - ' -c log_rotation_age=0' . - ' -c log_rotation_size=0' . - ' -c log_error_verbosity=verbose' . - ' -c unix_socket_directories=\'' . $self->dbPath() . '\'"' . - ' -D ' . $self->dbBasePath() . ' -l ' . $self->pgLogFile() . ' -s'; - - $self->executeSimple($strCommand); - - # Connect user session - $self->sqlConnect(); -} - -#################################################################################################################################### -# clusterStop -# -# Stop the PostgreSQL cluster and optionally check for errors in the server log. -#################################################################################################################################### -sub clusterStop -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $bIgnoreLogError = defined($$hParam{bIgnoreLogError}) ? $$hParam{bIgnoreLogError} : false; - my $bStop = defined($hParam->{bStop}) ? $$hParam{bStop} : true; - - # Disconnect user session - $self->sqlDisconnect(); - - # Grep for errors in postgresql.log - this is done first because we want to ignore any errors that happen during shutdown. - if (!$bIgnoreLogError && storageTest()->exists($self->pgLogFile())) - { - $self->executeSimple( - 'grep -v "FATAL\: 57P03\: the database system is (starting up|not yet accepting connections|" - "not accepting connections)" ' . $self->pgLogFile() . ' | grep "ERROR\|FATAL"', - {iExpectedExitStatus => 1}); - } - - # If pg process is running then stop the cluster - if ($bStop && -e $self->dbBasePath() . '/' . DB_FILE_POSTMTRPID) - { - $self->executeSimple($self->pgBinPath() . '/pg_ctl stop -D ' . $self->dbBasePath() . ' -w -s -m fast'); - } - - # Remove the log file - storageTest()->remove($self->pgLogFile(), {bIgnoreMissing => true}); -} - -#################################################################################################################################### -# clusterRestart -# -# Restart the PostgreSQL cluster. -#################################################################################################################################### -sub clusterRestart -{ - my $self = shift; - my $hParam = shift; - - $self->clusterStop($hParam); - $self->clusterStart($hParam); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub walId {return shift->pgVersion() >= PG_VERSION_10 ? 'wal' : 'xlog'} -sub pgBinPath {return testRunGet()->pgBinPath()} -sub pgLogFile {return shift->{strPgLogFile}} -sub pgLogPath {return shift->{strPgLogPath}} -sub pgPort {return shift->{iPgPort}} -sub pgSocketPath {return shift->{strPgSocketPath}} -sub pgVersion {return testRunGet()->pgVersion()} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm b/test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm deleted file mode 100644 index e9f89c159..000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm +++ /dev/null @@ -1,80 +0,0 @@ -#################################################################################################################################### -# GCS Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostGcsTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# GCS defaults -#################################################################################################################################### -use constant HOST_GCS_BUCKET => 'gcsbucket'; - push @EXPORT, qw(HOST_GCS_BUCKET); -use constant HOST_GCS_KEY => 'testkey'; - push @EXPORT, qw(HOST_GCS_KEY); -use constant HOST_GCS_KEY_TYPE => 'token'; - push @EXPORT, qw(HOST_GCS_KEY_TYPE); -use constant HOST_GCS_PORT => 4443; - push @EXPORT, qw(HOST_GCS_PORT); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert"; - - my $self = $class->SUPER::new( - HOST_GCS, 'test-' . testRunGet()->vmId() . '-' . HOST_GCS, 'fsouza/fake-gcs-server', 'root', undef, undef, undef, - false); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm deleted file mode 100644 index 274619cdc..000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm +++ /dev/null @@ -1,86 +0,0 @@ -#################################################################################################################################### -# S3 Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostS3Test; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# S3 defaults -#################################################################################################################################### -use constant HOST_S3_ACCESS_KEY => 'accessKey1'; - push @EXPORT, qw(HOST_S3_ACCESS_KEY); -use constant HOST_S3_ACCESS_SECRET_KEY => 'verySecretKey1'; - push @EXPORT, qw(HOST_S3_ACCESS_SECRET_KEY); -use constant HOST_S3_BUCKET => 'pgbackrest-dev'; - push @EXPORT, qw(HOST_S3_BUCKET); -use constant HOST_S3_ENDPOINT => 's3.amazonaws.com'; - push @EXPORT, qw(HOST_S3_ENDPOINT); -use constant HOST_S3_REGION => 'us-east-1'; - push @EXPORT, qw(HOST_S3_REGION); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert"; - - my $self = $class->SUPER::new( - HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', 'minio/minio:RELEASE.2023-09-30T07-02-29Z', 'root', - ["${strFakeCertPath}/s3-server.crt:/root/.minio/certs/public.crt:ro", - "${strFakeCertPath}/s3-server.key:/root/.minio/certs/private.key:ro"], - '-e MINIO_REGION=' . HOST_S3_REGION . ' -e MINIO_DOMAIN=' . HOST_S3_ENDPOINT . ' -e MINIO_BROWSER=off' . - ' -e MINIO_ACCESS_KEY=' . HOST_S3_ACCESS_KEY . ' -e MINIO_SECRET_KEY=' . HOST_S3_ACCESS_SECRET_KEY, - 'server /data --address :443', false); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm b/test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm deleted file mode 100644 index 83efedc6c..000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm +++ /dev/null @@ -1,72 +0,0 @@ -#################################################################################################################################### -# SFTP Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostSftpTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# SFTP defaults -#################################################################################################################################### -use constant HOST_SFTP_ACCOUNT => TEST_USER; - push @EXPORT, qw(HOST_SFTP_ACCOUNT); -use constant HOST_SFTP_HOSTKEY_HASH_TYPE => 'sha1'; - push @EXPORT, qw(HOST_SFTP_HOSTKEY_HASH_TYPE); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $self = $class->SUPER::new( - HOST_SFTP, 'test-' . testRunGet()->vmId() . '-' . HOST_SFTP, containerRepo() . ':' . testRunGet()->vm() . '-test', 'root'); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/HostEnvTest.pm b/test/lib/pgBackRestTest/Env/HostEnvTest.pm deleted file mode 100644 index e7ca012d9..000000000 --- a/test/lib/pgBackRestTest/Env/HostEnvTest.pm +++ /dev/null @@ -1,543 +0,0 @@ -#################################################################################################################################### -# FullCommonTest.pm - Common code for backup tests -#################################################################################################################################### -package pgBackRestTest::Env::HostEnvTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Digest::SHA qw(sha1_hex); -use Exporter qw(import); - our @EXPORT = qw(); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::Host::HostAzureTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::Host::HostDbSyntheticTest; -use pgBackRestTest::Env::Host::HostGcsTest; -use pgBackRestTest::Env::Host::HostS3Test; - -#################################################################################################################################### -# Constants -#################################################################################################################################### -use constant ENCRYPTION_KEY_ARCHIVE => 'archive'; - push @EXPORT, qw(ENCRYPTION_KEY_ARCHIVE); -use constant ENCRYPTION_KEY_MANIFEST => 'manifest'; - push @EXPORT, qw(ENCRYPTION_KEY_MANIFEST); -use constant ENCRYPTION_KEY_BACKUPSET => 'backupset'; - push @EXPORT, qw(ENCRYPTION_KEY_BACKUPSET); - -#################################################################################################################################### -# setup -#################################################################################################################################### -sub setup -{ - my $self = shift; - my $bSynthetic = shift; - my $oConfigParam = shift; - - # Start object server first since it takes the longest - #------------------------------------------------------------------------------------------------------------------------------- - my $oHostObject; - - if ($oConfigParam->{strStorage} eq S3) - { - $oHostObject = new pgBackRestTest::Env::Host::HostS3Test(); - } - elsif ($oConfigParam->{strStorage} eq AZURE) - { - $oHostObject = new pgBackRestTest::Env::Host::HostAzureTest(); - } - elsif ($oConfigParam->{strStorage} eq GCS) - { - $oHostObject = new pgBackRestTest::Env::Host::HostGcsTest(); - } - elsif ($oConfigParam->{strStorage} eq SFTP) - { - $oHostObject = new pgBackRestTest::Env::Host::HostSftpTest(); - } - - # Get host group - my $oHostGroup = hostGroupGet(); - - # Create the backup host - my $strBackupDestination; - my $bHostBackup = defined($$oConfigParam{bHostBackup}) ? $$oConfigParam{bHostBackup} : false; - my $oHostBackup = undef; - - my $bRepoEncrypt = defined($$oConfigParam{bRepoEncrypt}) ? $$oConfigParam{bRepoEncrypt} : false; - - if ($bHostBackup) - { - $strBackupDestination = defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_BACKUP; - - $oHostBackup = new pgBackRestTest::Env::Host::HostBackupTest( - {strBackupDestination => $strBackupDestination, bSynthetic => $bSynthetic, - bRepoLocal => $oConfigParam->{strStorage} eq POSIX, bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}}); - $oHostGroup->hostAdd($oHostBackup); - } - else - { - $strBackupDestination = - defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_DB_PRIMARY; - } - - # Create the db-primary host - my $oHostDbPrimary = undef; - - if ($bSynthetic) - { - $oHostDbPrimary = new pgBackRestTest::Env::Host::HostDbSyntheticTest( - {strBackupDestination => $strBackupDestination, - bRepoLocal => $oConfigParam->{strStorage} eq POSIX, bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}}); - } - else - { - $oHostDbPrimary = new pgBackRestTest::Env::Host::HostDbTest( - {strBackupDestination => $strBackupDestination, bRepoLocal => $oConfigParam->{strStorage} eq POSIX, - bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}}); - } - - $oHostGroup->hostAdd($oHostDbPrimary); - - # Create the db-standby host - my $oHostDbStandby = undef; - - if (defined($$oConfigParam{bStandby}) && $$oConfigParam{bStandby}) - { - $oHostDbStandby = new pgBackRestTest::Env::Host::HostDbTest( - {strBackupDestination => $strBackupDestination, bStandby => true, bRepoLocal => $oConfigParam->{strStorage} eq POSIX, - bTls => $oConfigParam->{bTls}}); - - $oHostGroup->hostAdd($oHostDbStandby); - } - - # Finalize object server - #------------------------------------------------------------------------------------------------------------------------------- - if ($oConfigParam->{strStorage} eq S3) - { - $oHostGroup->hostAdd($oHostObject, {rstryHostName => ['pgbackrest-dev.s3.amazonaws.com', 's3.amazonaws.com']}); - } - elsif ($oConfigParam->{strStorage} eq AZURE || $oConfigParam->{strStorage} eq GCS || $oConfigParam->{strStorage} eq SFTP) - { - $oHostGroup->hostAdd($oHostObject); - } - - # Create db-primary config - $oHostDbPrimary->configCreate({ - bTls => $oConfigParam->{bTls}, - strBackupSource => $$oConfigParam{strBackupSource}, - strCompressType => $$oConfigParam{strCompressType}, - bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink}, - bArchiveAsync => $$oConfigParam{bArchiveAsync}, - strStorage => $oConfigParam->{strStorage}, - iRepoTotal => $oConfigParam->{iRepoTotal}, - bBundle => $oConfigParam->{bBundle}, - bBlockIncr => $oConfigParam->{bBlockIncr}}); - - # Create backup config if backup host exists - if (defined($oHostBackup)) - { - $oHostBackup->configCreate({ - bTls => $oConfigParam->{bTls}, - strCompressType => $$oConfigParam{strCompressType}, - bHardlink => $$oConfigParam{bHardLink}, - strStorage => $oConfigParam->{strStorage}, - iRepoTotal => $oConfigParam->{iRepoTotal}, - bBundle => $oConfigParam->{bBundle}, - bBlockIncr => $oConfigParam->{bBlockIncr}}); - } - # If backup host is not defined set it to db-primary - else - { - $oHostBackup = $strBackupDestination eq HOST_DB_PRIMARY || $strBackupDestination eq HOST_SFTP ? $oHostDbPrimary : - $oHostDbStandby; - } - - storageRepoCommandSet( - $self->backrestExeHelper() . - ' --config=' . $oHostBackup->backrestConfig() . ' --stanza=' . $self->stanza() . ' --log-level-console=off' . - ' --log-level-stderr=error' . - ($oConfigParam->{strStorage} ne POSIX ? - ($oConfigParam->{strStorage} ne SFTP ? " --no-repo1-storage-verify-tls" : '') . - " --repo1-$oConfigParam->{strStorage}-" . - ($oConfigParam->{strStorage} eq GCS ? 'endpoint' : 'host') . "=" . $oHostObject->ipGet() : '') . - ($oConfigParam->{strStorage} eq GCS ? ':' . HOST_GCS_PORT : ''), - $oConfigParam->{strStorage} eq POSIX ? STORAGE_POSIX : STORAGE_OBJECT); - - # Create db-standby config - if (defined($oHostDbStandby)) - { - $oHostDbStandby->configCreate({ - bTls => $oConfigParam->{bTls}, - strBackupSource => $$oConfigParam{strBackupSource}, - strCompressType => $$oConfigParam{strCompressType}, - bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink}, - bArchiveAsync => $$oConfigParam{bArchiveAsync}, - strStorage => $oConfigParam->{strStorage}, - iRepoTotal => $oConfigParam->{iRepoTotal}, - bBundle => $oConfigParam->{bBundle}, - bBlockIncr => $oConfigParam->{bBlockIncr}}); - } - - # Create object storage - if (defined($oHostObject)) - { - storageRepo()->create(); - } - - return $oHostDbPrimary, $oHostDbStandby, $oHostBackup; -} - -#################################################################################################################################### -# Generate database system id for the db version -#################################################################################################################################### -sub dbSysId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbSysId', \@_, - {name => 'strPgVersion', trace => true}, - ); - - return (1000000000000000000 + ($strPgVersion * 10)); -} - -#################################################################################################################################### -# Get database catalog version for the db version -#################################################################################################################################### -sub dbCatalogVersion -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->sysId', \@_, - {name => 'strPgVersion', trace => true}, - ); - - my $hCatalogVersion = - { - &PG_VERSION_94 => 201409291, - &PG_VERSION_95 => 201510051, - &PG_VERSION_96 => 201608131, - &PG_VERSION_10 => 201707211, - &PG_VERSION_11 => 201806231, - &PG_VERSION_12 => 201909212, - &PG_VERSION_13 => 202007201, - &PG_VERSION_14 => 202105121, - &PG_VERSION_15 => 202209061, - &PG_VERSION_16 => 202307071, - }; - - if (!defined($hCatalogVersion->{$strPgVersion})) - { - confess &log(ASSERT, "no catalog version defined for pg version ${strPgVersion}"); - } - - return $hCatalogVersion->{$strPgVersion}; -} - -#################################################################################################################################### -# Generate control file content -#################################################################################################################################### -sub controlGenerateContent -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->controlGenerateContent', \@_, - {name => 'strPgVersion', trace => true}, - ); - - my $hControlContent = - { - 32 => - { - &PG_VERSION_94 => - "5e0064a7b3b6e00dae0300000b43010c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000002000000000000000000000000000010000000000000000000000000000000000000000" . - "000000006b0756c8", - &PG_VERSION_95 => - "5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "000000000000000000000000000000003bfe413a", - &PG_VERSION_96 => - "600064a7b3b6e00dc0030000c34b040c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "000000000000000000000000000000005d135da6", - &PG_VERSION_10 => - "640064a7b3b6e00dea030000cbce050c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8556c34", - }, - 64 => - { - &PG_VERSION_94 => - "5e0064a7b3b6e00dae0300000b43010c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "00000000000000000000000000000000ee6cf996", - &PG_VERSION_95 => - "5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" . - "0000000000000001000000000000000000000000000000000000000000000000381ec2de", - &PG_VERSION_96 => - "600064a7b3b6e00dc0030000c34b040c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" . - "00000000000000010000000000000000000000000000000000000000000000002d96a4c0", - &PG_VERSION_10 => - "640064a7b3b6e00dea030000cbce050c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" . - "0000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "00000000000000008d543cdf", - }, - }; - - my $strControlContent = $hControlContent->{$self->archBits()}{$strPgVersion}; - - if (!defined($strControlContent)) - { - confess &log(ASSERT, "no control content defined for pg version ${strPgVersion}"); - } - - my $tControlContent = ''; - - for (my $iIdx = 0; $iIdx < length($strControlContent) / 2; $iIdx++) - { - my $iChar = hex(substr($strControlContent, $iIdx * 2, 2)); - $tControlContent .= pack('C', $iChar); - } - - - # Pad bytes - for (my $iIdx = length($tControlContent); $iIdx < 8192; $iIdx++) - { - $tControlContent .= pack('C', 0); - } - - return \$tControlContent; -} - -#################################################################################################################################### -# Generate control file and write to disk -#################################################################################################################################### -sub controlGenerate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbPath, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->controlGenerate', \@_, - {name => 'strDbPath', trace => true}, - {name => 'strPgVersion', trace => true}, - ); - - storageTest()->put("${strDbPath}/global/pg_control", $self->controlGenerateContent($strPgVersion)); -} - -#################################################################################################################################### -# walSegment -# -# Generate name of WAL segment from component parts. -#################################################################################################################################### -sub walSegment -{ - my $self = shift; - my $iTimeline = shift; - my $iMajor = shift; - my $iMinor = shift; - - return uc(sprintf('%08x%08x%08x', $iTimeline, $iMajor, $iMinor)); -} - -#################################################################################################################################### -# Generate WAL file content -#################################################################################################################################### -sub walGenerateContent -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - $iSourceNo, - ) = - logDebugParam - ( - __PACKAGE__ . '->walGenerateContent', \@_, - {name => 'strPgVersion', trace => true}, - {name => 'iSourceNo', optional => true, default => 1, trace => true}, - ); - - # Get WAL magic for the PG version - my $hWalMagic = - { - &PG_VERSION_94 => hex('0xD07E'), - &PG_VERSION_95 => hex('0xD087'), - &PG_VERSION_96 => hex('0xD093'), - &PG_VERSION_10 => hex('0xD097'), - &PG_VERSION_11 => hex('0xD098'), - &PG_VERSION_12 => hex('0xD101'), - &PG_VERSION_13 => hex('0xD106'), - }; - - my $tWalContent = pack('S', $hWalMagic->{$strPgVersion}); - - # Indicate that the long header is present - $tWalContent .= pack('S', 2); - - # Add junk (H for header) for the bytes that won't be read by the tests - my $iOffset = 12 + (testRunGet()->archBits() / 8); - $tWalContent .= ('H' x $iOffset); - - # Add the system identifier - $tWalContent .= pack('Q', $self->dbSysId($strPgVersion)); - - # Add segment size - $tWalContent .= pack('L', PG_WAL_SEGMENT_SIZE); - - # Add the source number to produce WAL segments with different checksums - $tWalContent .= pack('S', $iSourceNo); - - # Pad out to the required size (B for body) - $tWalContent .= ('B' x (PG_WAL_SEGMENT_SIZE - length($tWalContent))); - - return \$tWalContent; -} - -#################################################################################################################################### -# Generate WAL file content checksum -#################################################################################################################################### -sub walGenerateContentChecksum -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - $hParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->walGenerateContent', \@_, - {name => 'strPgVersion', trace => true}, - {name => 'hParam', required => false, trace => true}, - ); - - return sha1_hex(${$self->walGenerateContent($strPgVersion, $hParam)}); -} - -#################################################################################################################################### -# walGenerate -# -# Generate a WAL segment and ready file for testing. -#################################################################################################################################### -sub walGenerate -{ - my $self = shift; - my $strWalPath = shift; - my $strPgVersion = shift; - my $iSourceNo = shift; - my $strWalSegment = shift; - my $bPartial = shift; - my $bChecksum = shift; - my $bReady = shift; - - my $rtWalContent = $self->walGenerateContent($strPgVersion, {iSourceNo => $iSourceNo}); - my $strWalFile = - "${strWalPath}/${strWalSegment}" . ($bChecksum ? '-' . sha1_hex($rtWalContent) : '') . - (defined($bPartial) && $bPartial ? '.partial' : ''); - - # Put the WAL segment and the ready file - storageTest()->put($strWalFile, $rtWalContent); - - if (!defined($bReady) || $bReady) - { - storageTest()->put("${strWalPath}/archive_status/${strWalSegment}.ready"); - } - - return $strWalFile; -} - -#################################################################################################################################### -# walRemove -# -# Remove WAL file and ready file. -#################################################################################################################################### -sub walRemove -{ - my $self = shift; - my $strWalPath = shift; - my $strWalFile = shift; - - storageTest()->remove("$self->{strWalPath}/${strWalFile}"); - storageTest()->remove("$self->{strWalPath}/archive_status/${strWalFile}.ready"); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/InfoCommon.pm b/test/lib/pgBackRestTest/Env/InfoCommon.pm deleted file mode 100644 index 22990a81e..000000000 --- a/test/lib/pgBackRestTest/Env/InfoCommon.pm +++ /dev/null @@ -1,32 +0,0 @@ -#################################################################################################################################### -# INFO MODULE -# Constants, variables and functions common to the info files -#################################################################################################################################### -package pgBackRestTest::Env::InfoCommon; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); - -#################################################################################################################################### -# DB section constants -#################################################################################################################################### -use constant INFO_BACKUP_SECTION_DB => 'db'; - push @EXPORT, qw(INFO_BACKUP_SECTION_DB); -use constant INFO_BACKUP_SECTION_DB_HISTORY => INFO_BACKUP_SECTION_DB . ':history'; - push @EXPORT, qw(INFO_BACKUP_SECTION_DB_HISTORY); - -#################################################################################################################################### -# History section constants -#################################################################################################################################### -use constant INFO_HISTORY_ID => 'id'; - push @EXPORT, qw(INFO_HISTORY_ID); -use constant INFO_DB_VERSION => 'version'; - push @EXPORT, qw(INFO_DB_VERSION); -use constant INFO_SYSTEM_ID => 'system-id'; - push @EXPORT, qw(INFO_SYSTEM_ID); - -1; diff --git a/test/lib/pgBackRestTest/Env/Manifest.pm b/test/lib/pgBackRestTest/Env/Manifest.pm deleted file mode 100644 index 564e6db65..000000000 --- a/test/lib/pgBackRestTest/Env/Manifest.pm +++ /dev/null @@ -1,1446 +0,0 @@ -#################################################################################################################################### -# MANIFEST MODULE -#################################################################################################################################### -package pgBackRestTest::Env::Manifest; -use parent 'pgBackRestDoc::Common::Ini'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname basename); -use Fcntl qw(:mode); -use Time::Local qw(timelocal); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# File/path constants -#################################################################################################################################### -use constant PATH_BACKUP_HISTORY => 'backup.history'; - push @EXPORT, qw(PATH_BACKUP_HISTORY); -use constant FILE_MANIFEST => 'backup.manifest'; - push @EXPORT, qw(FILE_MANIFEST); -use constant FILE_MANIFEST_COPY => FILE_MANIFEST . INI_COPY_EXT; - push @EXPORT, qw(FILE_MANIFEST_COPY); - -#################################################################################################################################### -# Default match factor -#################################################################################################################################### -use constant MANIFEST_DEFAULT_MATCH_FACTOR => 0.1; - push @EXPORT, qw(MANIFEST_DEFAULT_MATCH_FACTOR); - -#################################################################################################################################### -# MANIFEST Constants -#################################################################################################################################### -use constant MANIFEST_TARGET_PGDATA => 'pg_data'; - push @EXPORT, qw(MANIFEST_TARGET_PGDATA); -use constant MANIFEST_TARGET_PGTBLSPC => 'pg_tblspc'; - push @EXPORT, qw(MANIFEST_TARGET_PGTBLSPC); - -use constant MANIFEST_VALUE_PATH => 'path'; - push @EXPORT, qw(MANIFEST_VALUE_PATH); -use constant MANIFEST_VALUE_LINK => 'link'; - push @EXPORT, qw(MANIFEST_VALUE_LINK); - -# Manifest sections -use constant MANIFEST_SECTION_BACKUP => 'backup'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP); -use constant MANIFEST_SECTION_BACKUP_DB => 'backup:db'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_DB); -use constant MANIFEST_SECTION_BACKUP_INFO => 'backup:info'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_INFO); -use constant MANIFEST_SECTION_BACKUP_OPTION => 'backup:option'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_OPTION); -use constant MANIFEST_SECTION_BACKUP_TARGET => 'backup:target'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_TARGET); -use constant MANIFEST_SECTION_DB => 'db'; - push @EXPORT, qw(MANIFEST_SECTION_DB); -use constant MANIFEST_SECTION_TARGET_PATH => 'target:path'; - push @EXPORT, qw(MANIFEST_SECTION_TARGET_PATH); -use constant MANIFEST_SECTION_TARGET_FILE => 'target:file'; - push @EXPORT, qw(MANIFEST_SECTION_TARGET_FILE); -use constant MANIFEST_SECTION_TARGET_LINK => 'target:link'; - push @EXPORT, qw(MANIFEST_SECTION_TARGET_LINK); - -# Backup metadata required for restores -use constant MANIFEST_KEY_ARCHIVE_START => 'backup-archive-start'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_START); -use constant MANIFEST_KEY_ARCHIVE_STOP => 'backup-archive-stop'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_STOP); -use constant MANIFEST_KEY_LABEL => 'backup-label'; - push @EXPORT, qw(MANIFEST_KEY_LABEL); -use constant MANIFEST_KEY_LSN_START => 'backup-lsn-start'; - push @EXPORT, qw(MANIFEST_KEY_LSN_START); -use constant MANIFEST_KEY_LSN_STOP => 'backup-lsn-stop'; - push @EXPORT, qw(MANIFEST_KEY_LSN_STOP); -use constant MANIFEST_KEY_PRIOR => 'backup-prior'; - push @EXPORT, qw(MANIFEST_KEY_PRIOR); -use constant MANIFEST_KEY_TIMESTAMP_COPY_START => 'backup-timestamp-copy-start'; - push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_COPY_START); -use constant MANIFEST_KEY_TIMESTAMP_START => 'backup-timestamp-start'; - push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_START); -use constant MANIFEST_KEY_TIMESTAMP_STOP => 'backup-timestamp-stop'; - push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_STOP); -use constant MANIFEST_KEY_TYPE => 'backup-type'; - push @EXPORT, qw(MANIFEST_KEY_TYPE); - -# Options that were set when the backup was made -use constant MANIFEST_KEY_BACKUP_STANDBY => 'option-backup-standby'; - push @EXPORT, qw(MANIFEST_KEY_BACKUP_STANDBY); -use constant MANIFEST_KEY_HARDLINK => 'option-hardlink'; - push @EXPORT, qw(MANIFEST_KEY_HARDLINK); -use constant MANIFEST_KEY_ARCHIVE_CHECK => 'option-archive-check'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_CHECK); -use constant MANIFEST_KEY_ARCHIVE_COPY => 'option-archive-copy'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_COPY); -use constant MANIFEST_KEY_BUFFER_SIZE => 'option-buffer-size'; - push @EXPORT, qw(MANIFEST_KEY_BUFFER_SIZE); -use constant MANIFEST_KEY_CHECKSUM_PAGE => 'option-checksum-page'; - push @EXPORT, qw(MANIFEST_KEY_CHECKSUM_PAGE); -use constant MANIFEST_KEY_COMPRESS => 'option-compress'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS); -use constant MANIFEST_KEY_COMPRESS_TYPE => 'option-compress-type'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS_TYPE); -use constant MANIFEST_KEY_COMPRESS_LEVEL => 'option-compress-level'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL); -use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK => 'option-compress-level-network'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL_NETWORK); -use constant MANIFEST_KEY_ONLINE => 'option-online'; - push @EXPORT, qw(MANIFEST_KEY_ONLINE); -use constant MANIFEST_KEY_DELTA => 'option-delta'; - push @EXPORT, qw(MANIFEST_KEY_DELTA); -use constant MANIFEST_KEY_PROCESS_MAX => 'option-process-max'; - push @EXPORT, qw(MANIFEST_KEY_PROCESS_MAX); - -# Information about the database that was backed up -use constant MANIFEST_KEY_DB_ID => 'db-id'; - push @EXPORT, qw(MANIFEST_KEY_DB_ID); -use constant MANIFEST_KEY_SYSTEM_ID => 'db-system-id'; - push @EXPORT, qw(MANIFEST_KEY_SYSTEM_ID); -use constant MANIFEST_KEY_CATALOG => 'db-catalog-version'; - push @EXPORT, qw(MANIFEST_KEY_CATALOG); -use constant MANIFEST_KEY_CONTROL => 'db-control-version'; - push @EXPORT, qw(MANIFEST_KEY_CONTROL); -use constant MANIFEST_KEY_DB_LAST_SYSTEM_ID => 'db-last-system-id'; - push @EXPORT, qw(MANIFEST_KEY_DB_LAST_SYSTEM_ID); -use constant MANIFEST_KEY_DB_VERSION => 'db-version'; - push @EXPORT, qw(MANIFEST_KEY_DB_VERSION); - -# Subkeys used for path/file/link info -use constant MANIFEST_SUBKEY_CHECKSUM => 'checksum'; - push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM); -use constant MANIFEST_SUBKEY_CHECKSUM_PAGE => 'checksum-page'; - push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE); -use constant MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR => 'checksum-page-error'; - push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR); -use constant MANIFEST_SUBKEY_DESTINATION => 'destination'; - push @EXPORT, qw(MANIFEST_SUBKEY_DESTINATION); -use constant MANIFEST_SUBKEY_FILE => 'file'; - push @EXPORT, qw(MANIFEST_SUBKEY_FILE); -use constant MANIFEST_SUBKEY_FUTURE => 'future'; - push @EXPORT, qw(MANIFEST_SUBKEY_FUTURE); -use constant MANIFEST_SUBKEY_GROUP => 'group'; - push @EXPORT, qw(MANIFEST_SUBKEY_GROUP); -use constant MANIFEST_SUBKEY_MODE => 'mode'; - push @EXPORT, qw(MANIFEST_SUBKEY_MODE); -use constant MANIFEST_SUBKEY_TIMESTAMP => 'timestamp'; - push @EXPORT, qw(MANIFEST_SUBKEY_TIMESTAMP); -use constant MANIFEST_SUBKEY_TYPE => 'type'; - push @EXPORT, qw(MANIFEST_SUBKEY_TYPE); -use constant MANIFEST_SUBKEY_PATH => 'path'; - push @EXPORT, qw(MANIFEST_SUBKEY_PATH); -use constant MANIFEST_SUBKEY_REFERENCE => 'reference'; - push @EXPORT, qw(MANIFEST_SUBKEY_REFERENCE); -use constant MANIFEST_SUBKEY_REPO_SIZE => 'repo-size'; - push @EXPORT, qw(MANIFEST_SUBKEY_REPO_SIZE); -use constant MANIFEST_SUBKEY_SIZE => 'size'; - push @EXPORT, qw(MANIFEST_SUBKEY_SIZE); -use constant MANIFEST_SUBKEY_TABLESPACE_ID => 'tablespace-id'; - push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_ID); -use constant MANIFEST_SUBKEY_TABLESPACE_NAME => 'tablespace-name'; - push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_NAME); -use constant MANIFEST_SUBKEY_USER => 'user'; - push @EXPORT, qw(MANIFEST_SUBKEY_USER); - -#################################################################################################################################### -# Database locations for important files/paths -#################################################################################################################################### -use constant DB_PATH_ARCHIVESTATUS => 'archive_status'; - push @EXPORT, qw(DB_PATH_ARCHIVESTATUS); -use constant DB_PATH_BASE => 'base'; - push @EXPORT, qw(DB_PATH_BASE); -use constant DB_PATH_GLOBAL => 'global'; - push @EXPORT, qw(DB_PATH_GLOBAL); -use constant DB_PATH_PGDYNSHMEM => 'pg_dynshmem'; - push @EXPORT, qw(DB_PATH_PGDYNSHMEM); -use constant DB_PATH_PGMULTIXACT => 'pg_multixact'; - push @EXPORT, qw(DB_PATH_PGMULTIXACT); -use constant DB_PATH_PGNOTIFY => 'pg_notify'; - push @EXPORT, qw(DB_PATH_PGNOTIFY); -use constant DB_PATH_PGREPLSLOT => 'pg_replslot'; - push @EXPORT, qw(DB_PATH_PGREPLSLOT); -use constant DB_PATH_PGSERIAL => 'pg_serial'; - push @EXPORT, qw(DB_PATH_PGSERIAL); -use constant DB_PATH_PGSNAPSHOTS => 'pg_snapshots'; - push @EXPORT, qw(DB_PATH_PGSNAPSHOTS); -use constant DB_PATH_PGSTATTMP => 'pg_stat_tmp'; - push @EXPORT, qw(DB_PATH_PGSTATTMP); -use constant DB_PATH_PGSUBTRANS => 'pg_subtrans'; - push @EXPORT, qw(DB_PATH_PGSUBTRANS); -use constant DB_PATH_PGTBLSPC => 'pg_tblspc'; - push @EXPORT, qw(DB_PATH_PGTBLSPC); - -use constant DB_FILE_BACKUPLABEL => 'backup_label'; - push @EXPORT, qw(DB_FILE_BACKUPLABEL); -use constant DB_FILE_BACKUPLABELOLD => DB_FILE_BACKUPLABEL . '.old'; - push @EXPORT, qw(DB_FILE_BACKUPLABELOLD); -use constant DB_FILE_PGCONTROL => DB_PATH_GLOBAL . '/pg_control'; - push @EXPORT, qw(DB_FILE_PGCONTROL); -use constant DB_FILE_PGFILENODEMAP => 'pg_filenode.map'; - push @EXPORT, qw(DB_FILE_PGFILENODEMAP); -use constant DB_FILE_PGINTERNALINIT => 'pg_internal.init'; - push @EXPORT, qw(DB_FILE_PGINTERNALINIT); -use constant DB_FILE_PGVERSION => 'PG_VERSION'; - push @EXPORT, qw(DB_FILE_PGVERSION); -use constant DB_FILE_POSTGRESQLAUTOCONFTMP => 'postgresql.auto.conf.tmp'; - push @EXPORT, qw(DB_FILE_POSTGRESQLAUTOCONFTMP); -use constant DB_FILE_POSTMTROPTS => 'postmas'.'ter.opts'; - push @EXPORT, qw(DB_FILE_POSTMTROPTS); -use constant DB_FILE_POSTMTRPID => 'postmas'.'ter.pid'; - push @EXPORT, qw(DB_FILE_POSTMTRPID); -use constant DB_FILE_RECOVERYCONF => 'recovery.conf'; - push @EXPORT, qw(DB_FILE_RECOVERYCONF); -use constant DB_FILE_RECOVERYSIGNAL => 'recovery.signal'; - push @EXPORT, qw(DB_FILE_RECOVERYSIGNAL); -use constant DB_FILE_RECOVERYDONE => 'recovery.done'; - push @EXPORT, qw(DB_FILE_RECOVERYDONE); -use constant DB_FILE_STANDBYSIGNAL => 'standby.signal'; - push @EXPORT, qw(DB_FILE_STANDBYSIGNAL); -use constant DB_FILE_TABLESPACEMAP => 'tablespace_map'; - push @EXPORT, qw(DB_FILE_TABLESPACEMAP); - -use constant DB_FILE_PREFIX_TMP => 'pgsql_tmp'; - push @EXPORT, qw(DB_FILE_PREFIX_TMP); - -#################################################################################################################################### -# Manifest locations for important files/paths -#################################################################################################################################### -use constant MANIFEST_PATH_BASE => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_BASE; - push @EXPORT, qw(MANIFEST_PATH_BASE); -use constant MANIFEST_PATH_GLOBAL => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_GLOBAL; - push @EXPORT, qw(MANIFEST_PATH_GLOBAL); -use constant MANIFEST_PATH_PGDYNSHMEM => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGDYNSHMEM; - push @EXPORT, qw(MANIFEST_PATH_PGDYNSHMEM); -use constant MANIFEST_PATH_PGMULTIXACT => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGMULTIXACT; - push @EXPORT, qw(MANIFEST_PATH_PGMULTIXACT); -use constant MANIFEST_PATH_PGNOTIFY => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGNOTIFY; - push @EXPORT, qw(MANIFEST_PATH_PGNOTIFY); -use constant MANIFEST_PATH_PGREPLSLOT => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGREPLSLOT; - push @EXPORT, qw(MANIFEST_PATH_PGREPLSLOT); -use constant MANIFEST_PATH_PGSERIAL => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSERIAL; - push @EXPORT, qw(MANIFEST_PATH_PGSERIAL); -use constant MANIFEST_PATH_PGSNAPSHOTS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSNAPSHOTS; - push @EXPORT, qw(MANIFEST_PATH_PGSNAPSHOTS); -use constant MANIFEST_PATH_PGSTATTMP => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSTATTMP; - push @EXPORT, qw(MANIFEST_PATH_PGSTATTMP); -use constant MANIFEST_PATH_PGSUBTRANS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSUBTRANS; - push @EXPORT, qw(MANIFEST_PATH_PGSUBTRANS); -use constant MANIFEST_PATH_PGTBLSPC => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC; - push @EXPORT, qw(MANIFEST_PATH_PGTBLSPC); - -use constant MANIFEST_FILE_BACKUPLABEL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABEL; - push @EXPORT, qw(MANIFEST_FILE_BACKUPLABEL); -use constant MANIFEST_FILE_BACKUPLABELOLD => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABELOLD; - push @EXPORT, qw(MANIFEST_FILE_BACKUPLABELOLD); -use constant MANIFEST_FILE_PGCONTROL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGCONTROL; - push @EXPORT, qw(MANIFEST_FILE_PGCONTROL); -use constant MANIFEST_FILE_POSTGRESQLAUTOCONFTMP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTGRESQLAUTOCONFTMP; - push @EXPORT, qw(MANIFEST_FILE_PGCONTROL); -use constant MANIFEST_FILE_POSTMTROPTS => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMTROPTS; - push @EXPORT, qw(MANIFEST_FILE_POSTMTROPTS); -use constant MANIFEST_FILE_POSTMTRPID => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMTRPID; - push @EXPORT, qw(MANIFEST_FILE_POSTMTRPID); -use constant MANIFEST_FILE_RECOVERYCONF => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYCONF; - push @EXPORT, qw(MANIFEST_FILE_RECOVERYCONF); -use constant MANIFEST_FILE_RECOVERYSIGNAL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYSIGNAL; - push @EXPORT, qw(MANIFEST_FILE_RECOVERYSIGNAL); -use constant MANIFEST_FILE_RECOVERYDONE => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYDONE; - push @EXPORT, qw(MANIFEST_FILE_RECOVERYDONE); -use constant MANIFEST_FILE_STANDBYSIGNAL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_STANDBYSIGNAL; - push @EXPORT, qw(MANIFEST_FILE_STANDBYSIGNAL); -use constant MANIFEST_FILE_TABLESPACEMAP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_TABLESPACEMAP; - push @EXPORT, qw(MANIFEST_FILE_TABLESPACEMAP); - -#################################################################################################################################### -# Minimum ID for a user object in postgres -#################################################################################################################################### -use constant DB_USER_OBJECT_MINIMUM_ID => 16384; - push @EXPORT, qw(DB_USER_OBJECT_MINIMUM_ID); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileName, - $bLoad, - $oStorage, - $strDbVersion, - $iDbCatalogVersion, - $strCipherPass, # Passphrase to open the manifest if encrypted - $strCipherPassSub, # Passphrase to encrypt the backup files - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strFileName', trace => true}, - {name => 'bLoad', optional => true, default => true, trace => true}, - {name => 'oStorage', optional => true, default => storageRepo(), trace => true}, - {name => 'strDbVersion', optional => true, trace => true}, - {name => 'iDbCatalogVersion', optional => true, trace => true}, - {name => 'strCipherPass', optional => true, redact => true}, - {name => 'strCipherPassSub', optional => true, redact => true}, - ); - - # Init object and store variables - my $self = $class->SUPER::new( - $oStorage, $strFileName, {bLoad => $bLoad, strCipherPass => $strCipherPass, strCipherPassSub => $strCipherPassSub}); - - # If manifest not loaded from a file then the db version and catalog version must be set - if (!$bLoad) - { - if (!(defined($strDbVersion) && defined($iDbCatalogVersion))) - { - confess &log(ASSERT, 'strDbVersion and iDbCatalogVersion must be provided with bLoad = false'); - } - - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION, undef, $strDbVersion . ''); - $self->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG, undef, $iDbCatalogVersion); - } - - # Mark the manifest as built if it was loaded from a file - $self->{bBuilt} = $bLoad; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# save -# -# Save the manifest. -#################################################################################################################################### -sub save -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->save'); - - # Call inherited save - $self->SUPER::save(); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# get -# -# Get a value. -#################################################################################################################################### -sub get -{ - my $self = shift; - my $strSection = shift; - my $strKey = shift; - my $strSubKey = shift; - my $bRequired = shift; - my $oDefault = shift; - - my $oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, false); - - if (!defined($oValue) && defined($strKey) && defined($strSubKey) && - ($strSection eq MANIFEST_SECTION_TARGET_FILE || $strSection eq MANIFEST_SECTION_TARGET_PATH || - $strSection eq MANIFEST_SECTION_TARGET_LINK) && - ($strSubKey eq MANIFEST_SUBKEY_USER || $strSubKey eq MANIFEST_SUBKEY_GROUP || $strSubKey eq MANIFEST_SUBKEY_MODE) && - $self->test($strSection, $strKey)) - { - $oValue = $self->SUPER::get("${strSection}:default", $strSubKey, undef, $bRequired, $oDefault); - } - else - { - $oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, $bRequired, $oDefault); - } - - return $oValue; -} - -#################################################################################################################################### -# boolGet -# -# Get a numeric value. -#################################################################################################################################### -sub boolGet -{ - my $self = shift; - my $strSection = shift; - my $strValue = shift; - my $strSubValue = shift; - my $bRequired = shift; - my $bDefault = shift; - - return $self->get($strSection, $strValue, $strSubValue, $bRequired, - defined($bDefault) ? ($bDefault ? INI_TRUE : INI_FALSE) : undef) ? true : false; -} - -#################################################################################################################################### -# numericGet -# -# Get a numeric value. -#################################################################################################################################### -sub numericGet -{ - my $self = shift; - my $strSection = shift; - my $strValue = shift; - my $strSubValue = shift; - my $bRequired = shift; - my $nDefault = shift; - - return $self->get($strSection, $strValue, $strSubValue, $bRequired, - defined($nDefault) ? $nDefault + 0 : undef) + 0; -} - -#################################################################################################################################### -# tablespacePathGet -# -# Get the unique path assigned by Postgres for the tablespace. -#################################################################################################################################### -sub tablespacePathGet -{ - my $self = shift; - - return('PG_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION) . - '_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG)); -} - -#################################################################################################################################### -# dbPathGet -# -# Convert a repo path to where the file actually belongs in the db. -#################################################################################################################################### -sub dbPathGet -{ - my $self = shift; - my $strDbPath = shift; - my $strFile = shift; - - my $strDbFile = defined($strDbPath) ? "${strDbPath}/" : ''; - - if (index($strFile, MANIFEST_TARGET_PGDATA . '/') == 0) - { - $strDbFile .= substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1); - } - else - { - $strDbFile .= $strFile; - } - - return $strDbFile; -} - -#################################################################################################################################### -# repoPathGet -# -# Convert a database path to where to file is located in the repo. -#################################################################################################################################### -sub repoPathGet -{ - my $self = shift; - my $strTarget = shift; - my $strFile = shift; - - my $strRepoFile = $strTarget; - - if ($self->isTargetTablespace($strTarget)) - { - $strRepoFile .= '/' . $self->tablespacePathGet(); - } - - if (defined($strFile)) - { - $strRepoFile .= "/${strFile}"; - } - - return $strRepoFile; -} - -#################################################################################################################################### -# isTargetValid -# -# Determine if a target is valid. -#################################################################################################################################### -sub isTargetValid -{ - my $self = shift; - my $strTarget = shift; - my $bError = shift; - - if (!defined($strTarget)) - { - confess &log(ASSERT, 'target is not defined'); - } - - if (!$self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget)) - { - if (defined($bError) && $bError) - { - confess &log(ASSERT, "${strTarget} is not a valid target"); - } - - return false; - } - - return true; -} - -#################################################################################################################################### -# isTargetLink -# -# Determine if a target is a link. -#################################################################################################################################### -sub isTargetLink -{ - my $self = shift; - my $strTarget = shift; - - $self->isTargetValid($strTarget, true); - - return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TYPE, MANIFEST_VALUE_LINK); -} - -#################################################################################################################################### -# isTargetFile -# -# Determine if a target is a file link. -#################################################################################################################################### -sub isTargetFile -{ - my $self = shift; - my $strTarget = shift; - - $self->isTargetValid($strTarget, true); - - return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE); -} - -#################################################################################################################################### -# isTargetTablespace -# -# Determine if a target is a tablespace. -#################################################################################################################################### -sub isTargetTablespace -{ - my $self = shift; - my $strTarget = shift; - - $self->isTargetValid($strTarget, true); - - return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TABLESPACE_ID); -} - -#################################################################################################################################### -# checkDelta -# -# Determine if the delta option should be enabled. Only called if delta has not yet been enabled. -#################################################################################################################################### -sub checkDelta -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strLastBackupSource, - $bOnlineSame, - $strTimelineCurrent, - $strTimelineLast, - ) = - logDebugParam - ( - __PACKAGE__ . '->checkDelta', \@_, - {name => 'strLastBackupSource'}, - {name => 'bOnlineSame'}, - {name => 'strTimelineCurrent', required => false}, - {name => 'strTimelineLast', required => false}, - ); - - my $bDelta = false; - - # Determine if a timeline switch has occurred - if (defined($strTimelineLast) && defined($strTimelineCurrent)) - { - # If there is a prior backup, check if a timeline switch has occurred since then - if ($strTimelineLast ne $strTimelineCurrent) - { - &log(WARN, "a timeline switch has occurred since the ${strLastBackupSource} backup, enabling delta checksum"); - $bDelta = true; - } - } - - # If delta was not set above and there is a change in the online option, then set delta option - if (!$bDelta && !$bOnlineSame) - { - &log(WARN, "the online option has changed since the ${strLastBackupSource} backup, enabling delta checksum"); - $bDelta = true; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bDelta', value => $bDelta, trace => true}, - ); -} - -#################################################################################################################################### -# checkDeltaFile -# -# Determine if the delta option should be enabled. Only called if delta has not yet been enabled. -#################################################################################################################################### -sub checkDeltaFile -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $stryFileList, - $oPriorManifest, - $lTimeBegin, - ) = - logDebugParam - ( - __PACKAGE__ . '->checkDeltaFile', \@_, - {name => 'stryFileList'}, - {name => 'oPriorManifest', required => false}, - {name => 'lTimeBegin', required => false}, - ); - - my $bDelta = false; - - # Loop though all files - foreach my $strName (@{$stryFileList}) - { - # If $lTimeBegin is defined, then this is not an aborted manifest so check if modification time is in the future (in this - # backup OR the last backup) then enable delta and exit - if (defined($lTimeBegin) && - ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin || - (defined($oPriorManifest) && - $oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y')))) - { - &log(WARN, "file $strName has timestamp in the future, enabling delta checksum"); - $bDelta = true; - last; - } - - # If the time on the file is earlier than the last manifest time or the size is different but the timestamp is the - # same, then enable delta and exit - if (defined($oPriorManifest) && $oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) && - ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) < - $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) || - ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != - $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) && - $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) == - $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP)))) - { - &log(WARN, "file $strName timestamp in the past or size changed but timestamp did not, enabling delta checksum"); - $bDelta = true; - last; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bDelta', value => $bDelta, trace => true}, - ); -} - -#################################################################################################################################### -# build -# -# Build the manifest object. -#################################################################################################################################### -sub build -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorageDbPrimary, - $strPath, - $oLastManifest, - $bOnline, - $bDelta, - $hTablespaceMap, - $hDatabaseMap, - $rhExclude, - $strTimelineCurrent, - $strTimelineLast, - $strLevel, - $bTablespace, - $strParentPath, - $strFilter, - $iLevel, - ) = - logDebugParam - ( - __PACKAGE__ . '->build', \@_, - {name => 'oStorageDbPrimary'}, - {name => 'strPath'}, - {name => 'oLastManifest', required => false}, - {name => 'bOnline'}, - {name => 'bDelta'}, - {name => 'hTablespaceMap', required => false}, - {name => 'hDatabaseMap', required => false}, - {name => 'rhExclude', required => false}, - {name => 'strTimelineCurrent', required => false}, - {name => 'strTimelineLast', required => false}, - {name => 'strLevel', required => false}, - {name => 'bTablespace', required => false}, - {name => 'strParentPath', required => false}, - {name => 'strFilter', required => false}, - {name => 'iLevel', required => false, default => 0}, - ); - - # Limit recursion to something reasonable (if more then we are very likely in a link loop) - if ($iLevel >= 16) - { - confess &log( - ERROR, - "recursion in manifest build exceeds depth of ${iLevel}: ${strLevel}\n" . - 'HINT: is there a link loop in $PGDATA?', - ERROR_FORMAT); - } - - if (!defined($strLevel)) - { - # Don't allow the manifest to be built more than once - if ($self->{bBuilt}) - { - confess &log(ASSERT, "manifest has already been built"); - } - - $self->{bBuilt} = true; - - # Set initial level - $strLevel = MANIFEST_TARGET_PGDATA; - - # If not online then build the tablespace map from pg_tblspc path - if (!$bOnline && !defined($hTablespaceMap)) - { - my $hTablespaceManifest = $oStorageDbPrimary->manifest($strPath . '/' . DB_PATH_PGTBLSPC); - $hTablespaceMap = {}; - - foreach my $strOid (sort(CORE::keys(%{$hTablespaceManifest}))) - { - if ($strOid eq '.' or $strOid eq '..') - { - next; - } - - logDebugMisc($strOperation, "found tablespace ${strOid} in offline mode"); - - $hTablespaceMap->{$strOid} = "ts${strOid}"; - } - } - - # If there is a last manifest, then check to see if delta checksum should be enabled - if (defined($oLastManifest) && !$bDelta) - { - $bDelta = $self->checkDelta( - 'last', $oLastManifest->boolTest(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, $bOnline), - $strTimelineCurrent, $strTimelineLast); - } - } - - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH, $strPath); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TYPE, - $strLevel eq MANIFEST_TARGET_PGDATA ? MANIFEST_VALUE_PATH : MANIFEST_VALUE_LINK); - - if ($bTablespace) - { - my $iTablespaceId = (split('\/', $strLevel))[1]; - - if (!defined($hTablespaceMap->{$iTablespaceId})) - { - confess &log(ASSERT, "tablespace with oid ${iTablespaceId} not found in tablespace map\n" . - "HINT: was a tablespace created or dropped during the backup?"); - } - - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_ID, $iTablespaceId); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_NAME, - $hTablespaceMap->{$iTablespaceId}); - } - - if (index($strPath, '/') != 0) - { - if (!defined($strParentPath)) - { - confess &log(ASSERT, "cannot get manifest for '${strPath}' when no parent path is specified"); - } - - $strPath = $oStorageDbPrimary->pathAbsolute($strParentPath, $strPath); - } - - # Get the manifest for this level - my $hManifest = $oStorageDbPrimary->manifest($strPath, {strFilter => $strFilter}); - my $strManifestType = MANIFEST_VALUE_LINK; - - # Loop though all paths/files/links in the manifest - foreach my $strName (sort(CORE::keys(%{$hManifest}))) - { - my $strFile = $strLevel; - - if ($strName ne '.') - { - if ($strManifestType eq MANIFEST_VALUE_LINK && $hManifest->{$strName}{type} eq 'l') - { - confess &log(ERROR, 'link \'' . - $self->dbPathGet( - $self->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH), $strLevel) . - '\' -> \'' . $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH) . - '\' cannot reference another link', ERROR_LINK_DESTINATION); - } - - if ($strManifestType eq MANIFEST_VALUE_LINK) - { - $strFile = dirname($strFile); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH, - dirname($self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH))); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_FILE, $strName); - } - - $strFile .= "/${strName}"; - } - else - { - $strManifestType = MANIFEST_VALUE_PATH; - } - - # Skip wal directory when doing an online backup. WAL will be restored from the archive or stored in the wal directory at - # the end of the backup if the archive-copy option is set. - next if ($bOnline && $strFile =~ (qw{^} . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . '\/') && - $strFile !~ ('^' . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . qw{/} . DB_PATH_ARCHIVESTATUS . '$')); - - # Skip all directories and files that start with pgsql_tmp. The files are removed when the server is restarted and the - # directories are recreated. - next if $strName =~ ('(^|\/)' . DB_FILE_PREFIX_TMP); - - # Skip pg_dynshmem/* since these files cannot be reused on recovery - next if $strFile =~ ('^' . MANIFEST_PATH_PGDYNSHMEM . '\/') && $self->dbVersion() >= PG_VERSION_94; - - # Skip pg_notify/* since these files cannot be reused on recovery - next if $strFile =~ ('^' . MANIFEST_PATH_PGNOTIFY . '\/'); - - # Skip pg_replslot/* since these files are generally not useful after a restore - next if $strFile =~ ('^' . MANIFEST_PATH_PGREPLSLOT . '\/') && $self->dbVersion() >= PG_VERSION_94; - - # Skip pg_serial/* since these files are reset - next if $strFile =~ ('^' . MANIFEST_PATH_PGSERIAL . '\/'); - - # Skip pg_snapshots/* since these files cannot be reused on recovery - next if $strFile =~ ('^' . MANIFEST_PATH_PGSNAPSHOTS . '\/'); - - # Skip temporary statistics in pg_stat_tmp even when stats_temp_directory is set because PGSS_TEXT_FILE is always created - # there. - next if $strFile =~ ('^' . MANIFEST_PATH_PGSTATTMP . '\/'); - - # Skip pg_subtrans/* since these files are reset - next if $strFile =~ ('^' . MANIFEST_PATH_PGSUBTRANS . '\/'); - - # Skip pg_internal.init since it is recreated on startup - next if $strFile =~ (DB_FILE_PGINTERNALINIT . '$'); - - # Skip recovery files - if ($self->dbVersion() >= PG_VERSION_12) - { - next if ($strFile eq MANIFEST_FILE_RECOVERYSIGNAL || $strFile eq MANIFEST_FILE_STANDBYSIGNAL); - } - else - { - next if ($strFile eq MANIFEST_FILE_RECOVERYDONE || $strFile eq MANIFEST_FILE_RECOVERYCONF); - } - - # Skip ignored files - if ($strFile eq MANIFEST_FILE_POSTGRESQLAUTOCONFTMP || # postgresql.auto.conf.tmp - temp file for safe writes - $strFile eq MANIFEST_FILE_BACKUPLABELOLD || # backup_label.old - old backup labels are not useful - $strFile eq MANIFEST_FILE_POSTMTROPTS || # not useful for backup - $strFile eq MANIFEST_FILE_POSTMTRPID) # to avoid confusing postgres after restore - { - next; - } - - # Check for files to exclude - if ($hManifest->{$strName}{type} eq 'f') - { - # Get the directory name from the manifest; it will be used later to search for existence in the keys - my $strDir = dirname($strName); - - # If it is a database data directory (base or tablespace) then check for files to skip - if ($strDir =~ '^base\/[0-9]+$' || - $strDir =~ ('^' . $self->tablespacePathGet() . '\/[0-9]+$')) - { - # Get just the filename - my $strBaseName = basename($strName); - - # Skip temp tables (lower t followed by numbers underscore numbers and a dot (segment) or underscore (fork) and/or - # segment, e.g. t1234_123, t1234_123.1, t1234_123_vm, t1234_123_fsm.1 - if ($strBaseName =~ '^t[0-9]+\_[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$') - { - next; - } - - # Check for unlogged tables to skip - # Exclude all forks for unlogged tables except the init fork (numbers underscore init and optional dot segment) - if ($strBaseName =~ '^[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$') - { - # Get the filenode (OID) - my ($strFileNode) = $strBaseName =~ '^(\d+)'; - - # Add _init to the OID to see if this is an unlogged object - $strFileNode = $strDir. "/" . $strFileNode . "_init"; - - # If key exists in manifest then skip - if (exists($hManifest->{$strFileNode}) && $hManifest->{$strFileNode}{type} eq 'f') - { - next; - } - } - } - } - - # Exclude files requested by the user - if (defined($rhExclude)) - { - # Exclusions are based on the name of the file relative to PGDATA - my $strPgFile = $self->dbPathGet(undef, $strFile); - my $bExclude = false; - - # Iterate through exclusions - foreach my $strExclude (sort(keys(%{$rhExclude}))) - { - # If the exclusion ends in / then we must do a prefix match - if ($strExclude =~ /\/$/) - { - if (index($strPgFile, $strExclude) == 0) - { - $bExclude = true; - } - } - # Else an exact match or a prefix match with / appended is required - elsif ($strPgFile eq $strExclude || index($strPgFile, "${strExclude}/") == 0) - { - $bExclude = true; - } - - # Log everything that gets excluded at a high level so it will hopefully be seen if wrong - if ($bExclude) - { - &log(INFO, "exclude ${strPgFile} from backup using '${strExclude}' exclusion"); - last; - } - } - - # Skip the file if it was excluded - next if $bExclude; - } - - my $cType = $hManifest->{$strName}{type}; - my $strSection = MANIFEST_SECTION_TARGET_PATH; - - if ($cType eq 'f') - { - $strSection = MANIFEST_SECTION_TARGET_FILE; - } - elsif ($cType eq 'l') - { - $strSection = MANIFEST_SECTION_TARGET_LINK; - } - elsif ($cType ne 'd') - { - &log(WARN, "exclude special file '" . $self->dbPathGet(undef, $strFile) . "' from backup"); - next; - } - - # Make sure that DB_PATH_PGTBLSPC contains only absolute links that do not point inside PGDATA - my $bTablespace = false; - - if (index($strName, DB_PATH_PGTBLSPC . '/') == 0 && $strLevel eq MANIFEST_TARGET_PGDATA) - { - $bTablespace = true; - $strFile = MANIFEST_TARGET_PGDATA . '/' . $strName; - - # Check for files in DB_PATH_PGTBLSPC that are not links - if ($hManifest->{$strName}{type} ne 'l') - { - confess &log(ERROR, "${strName} is not a symlink - " . DB_PATH_PGTBLSPC . ' should contain only symlinks', - ERROR_LINK_EXPECTED); - } - - # Check for tablespaces in PGDATA - if (index($hManifest->{$strName}{link_destination}, "${strPath}/") == 0 || - (index($hManifest->{$strName}{link_destination}, '/') != 0 && - index($oStorageDbPrimary->pathAbsolute($strPath . '/' . DB_PATH_PGTBLSPC, - $hManifest->{$strName}{link_destination}) . '/', "${strPath}/") == 0)) - { - confess &log(ERROR, 'tablespace symlink ' . $hManifest->{$strName}{link_destination} . - ' destination must not be in $PGDATA', ERROR_LINK_DESTINATION); - } - } - - # User and group required for all types - if (defined($hManifest->{$strName}{user})) - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_USER, $hManifest->{$strName}{user}); - } - else - { - $self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_USER, false); - } - - if (defined($hManifest->{$strName}{group})) - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_GROUP, $hManifest->{$strName}{group}); - } - else - { - $self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_GROUP, false); - } - - # Mode for required file and path type only - if ($cType eq 'f' || $cType eq 'd') - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_MODE, $hManifest->{$strName}{mode}); - } - - # Modification time and size required for file type only - if ($cType eq 'f') - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_TIMESTAMP, - $hManifest->{$strName}{modification_time} + 0); - $self->set($strSection, $strFile, MANIFEST_SUBKEY_SIZE, $hManifest->{$strName}{size} + 0); - } - - # Link destination required for link type only - if ($cType eq 'l') - { - my $strLinkDestination = $hManifest->{$strName}{link_destination}; - $self->set($strSection, $strFile, MANIFEST_SUBKEY_DESTINATION, $strLinkDestination); - - # If this is a tablespace then set the filter to use for the next level - my $strFilter; - - if ($bTablespace) - { - $strFilter = $self->tablespacePathGet(); - - $self->set(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGTBLSPC, undef, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA)); - - # PGDATA prefix was only needed for the link so strip it off before recursing - $strFile = substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1); - } - - $bDelta = $self->build( - $oStorageDbPrimary, $strLinkDestination, undef, $bOnline, $bDelta, $hTablespaceMap, $hDatabaseMap, $rhExclude, - undef, undef, $strFile, $bTablespace, dirname("${strPath}/${strName}"), $strFilter, $iLevel + 1); - } - } - - # If this is the base level then do post-processing - if ($strLevel eq MANIFEST_TARGET_PGDATA) - { - my $bTimeInFuture = false; - - # Wait for the remainder of the second when doing online backups. This is done because most filesystems only have a one - # second resolution and Postgres will still be modifying files during the second that the manifest is built and this could - # lead to an invalid diff/incr backup later when using timestamps to determine which files have changed. Offline backups do - # not wait because it makes testing much faster and Postgres should not be running (if it is the backup will not be - # consistent anyway and the one-second resolution problem is the least of our worries). - my $lTimeBegin = waitRemainder($bOnline); - - if (defined($oLastManifest)) - { - $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, - $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); - } - - # Store database map information when provided during an online backup. - foreach my $strDbName (sort(keys(%{$hDatabaseMap}))) - { - $self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_ID, - $hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_ID}); - $self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_LAST_SYSTEM_ID, - $hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_LAST_SYSTEM_ID}); - } - - # Determine if delta checksum should be enabled - if (!$bDelta) - { - my @stryFileList = $self->keys(MANIFEST_SECTION_TARGET_FILE); - - if (@stryFileList) - { - $bDelta = $self->checkDeltaFile(\@stryFileList, $oLastManifest, $lTimeBegin); - } - } - - # Loop though all files - foreach my $strName ($self->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # If modification time is in the future (in this backup OR the last backup) set warning flag and do not - # allow a reference - if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin || - (defined($oLastManifest) && - $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y'))) - { - $bTimeInFuture = true; - - # Only mark as future if still in the future in the current backup - if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y'); - } - } - # Else check if the size and timestamp match OR if the size matches and the delta option is set, then keep the file. - # In the latter case, if there had been a timestamp change then rather than removing and recopying the file, the file - # will be tested in backupFile to see if the db/repo checksum still matches: if so, it is not necessary to recopy, - # else it will need to be copied to the new backup. For zero sized files, the reference will be set and copying - # will be skipped later. - elsif (defined($oLastManifest) && $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) && - $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) == - $oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) && - ($bDelta || ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) == 0 || - $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) == - $oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP)))) - { - # Copy reference from previous backup if possible - if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE)) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE)); - } - # Otherwise the reference is to the previous backup - else - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE, - $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); - } - - # Copy the checksum from previous manifest (if it exists - zero sized files don't have checksums) - if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)); - } - - # Copy repo size from the previous manifest (if it exists) - if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE)) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE)); - } - - # Copy checksum page from the previous manifest (if it exists) - my $bChecksumPage = $oLastManifest->get( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, false); - - if (defined($bChecksumPage)) - { - $self->boolSet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, $bChecksumPage); - - if (!$bChecksumPage && - $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR)) - { - $self->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR)); - } - } - } - } - - # Warn if any files in the current backup are in the future - if ($bTimeInFuture) - { - &log(WARN, "some files have timestamps in the future - they will be copied to prevent possible race conditions"); - } - - # Record the time when copying will start - $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, $lTimeBegin + ($bOnline ? 1 : 0)); - - # Build default sections - $self->buildDefault(); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bDelta', value => $bDelta, trace => true}, - ); -} - -#################################################################################################################################### -# fileAdd -# -# Add files to the manifest that were generated after the initial manifest build, e.g. backup_label, tablespace_map, and copied WAL -# files. Since the files were not in the original cluster the user, group, and mode must be defaulted. -#################################################################################################################################### -sub fileAdd -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strManifestFile, - $lModificationTime, - $lSize, - $strChecksum, - $bPrimary, - ) = - logDebugParam - ( - __PACKAGE__ . '->fileAdd', \@_, - {name => 'strManifestFile'}, - {name => 'lModificationTime'}, - {name => 'lSize'}, - {name => 'lChecksum'}, - {name => 'bPrimary'}, - ); - - # Set manifest values - if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER) || - !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER, undef, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER))) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_USER, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER)); - } - - if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP) || - !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP, undef, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP))) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_GROUP, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP)); - } - - if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE) || - !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE, undef, '0600')) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_MODE, '0600'); - } - - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_TIMESTAMP, $lModificationTime); - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_SIZE, $lSize); - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_CHECKSUM, $strChecksum); -} - -#################################################################################################################################### -# buildDefault -# -# Builds the default section. -#################################################################################################################################### -sub buildDefault -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->buildDefault'); - - # Defaults for subkeys that tend to repeat - my $strDefaultUser = $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER); - my $strDefaultGroup = $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP); - my $strDefaultPathMode = $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_MODE); - my $strDefaultFileMode = sprintf('%04o', oct($strDefaultPathMode) & (S_IRUSR | S_IWUSR | S_IRGRP)); - - # Remove subkeys that match the defaults - foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK) - { - next if !$self->test($strSection); - - foreach my $strFile ($self->keys($strSection)) - { - if ($self->test($strSection, $strFile, MANIFEST_SUBKEY_USER, $strDefaultUser)) - { - $self->remove($strSection, $strFile, MANIFEST_SUBKEY_USER); - } - - if ($self->test($strSection, $strFile, MANIFEST_SUBKEY_GROUP, $strDefaultGroup)) - { - $self->remove($strSection, $strFile, MANIFEST_SUBKEY_GROUP); - } - - if ($self->test( - $strSection, $strFile, MANIFEST_SUBKEY_MODE, - $strSection eq MANIFEST_SECTION_TARGET_PATH ? $strDefaultPathMode : $strDefaultFileMode)) - { - $self->remove($strSection, $strFile, MANIFEST_SUBKEY_MODE); - } - } - } - - # Write defaults - $self->set(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER, undef, $strDefaultUser); - $self->set(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP, undef, $strDefaultGroup); - $self->set(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE, undef, $strDefaultFileMode); - - if ($self->test(MANIFEST_SECTION_TARGET_LINK)) - { - $self->set(MANIFEST_SECTION_TARGET_LINK . ':default', MANIFEST_SUBKEY_USER, undef, $strDefaultUser); - $self->set(MANIFEST_SECTION_TARGET_LINK . ':default', MANIFEST_SUBKEY_GROUP, undef, $strDefaultGroup); - } - - $self->set(MANIFEST_SECTION_TARGET_PATH . ':default', MANIFEST_SUBKEY_USER, undef, $strDefaultUser); - $self->set(MANIFEST_SECTION_TARGET_PATH . ':default', MANIFEST_SUBKEY_GROUP, undef, $strDefaultGroup); - $self->set(MANIFEST_SECTION_TARGET_PATH . ':default', MANIFEST_SUBKEY_MODE, undef, $strDefaultPathMode); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# validate -# -# Checks for any missing values or inconsistencies in the manifest. -#################################################################################################################################### -sub validate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . 'validate'); - - # Make sure that all files have size and checksum (when size > 0). Since these values are removed before the backup file copy - # starts this ensures that all files had results stored in the manifest during the file copy. - foreach my $strFile ($self->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # Ensure size is set - if (!$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE)) - { - confess &log(ASSERT, "manifest subvalue 'size' not set for file '${strFile}'"); - } - - # If size > 0 then checksum must also be set - if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE) > 0 && - !$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_CHECKSUM)) - { - confess &log(ASSERT, "manifest subvalue 'checksum' not set for file '${strFile}'"); - } - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# dbVersion - version of PostgreSQL that the manifest is being built for -#################################################################################################################################### -sub dbVersion -{ - my $self = shift; - - return $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION); -} - -#################################################################################################################################### -# xactPath - return the transaction directory based on the PostgreSQL version -#################################################################################################################################### -sub xactPath -{ - my $self = shift; - - return $self->dbVersion() >= PG_VERSION_10 ? 'pg_xact' : 'pg_clog'; -} - -#################################################################################################################################### -# walPath - return the wal directory based on the PostgreSQL version -#################################################################################################################################### -sub walPath -{ - my $self = shift; - - return $self->dbVersion() >= PG_VERSION_10 ? 'pg_wal' : 'pg_xlog'; -} - -#################################################################################################################################### -# isPrimaryFile -# -# Is this file required to be copied from the primary? -#################################################################################################################################### -sub isPrimaryFile -{ - my $self = shift; - my $strFile = shift; - - return - $strFile !~ ('^(' . MANIFEST_TARGET_PGDATA . '\/' . '(' . DB_PATH_BASE . '|' . DB_PATH_GLOBAL . '|' . - $self->xactPath() . '|' . DB_PATH_PGMULTIXACT . ')|' . DB_PATH_PGTBLSPC . ')\/'); -} - -#################################################################################################################################### -# isChecksumPage -# -# Can this file have page checksums? -#################################################################################################################################### -sub isChecksumPage -{ - my $strFile = shift; - - if (($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_BASE . '\/[0-9]+\/|^' . MANIFEST_TARGET_PGTBLSPC . - '\/[0-9]+\/[^\/]+\/[0-9]+\/') && - $strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . ')$')) || - ($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_GLOBAL . '\/') && - $strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . '|' . - DB_FILE_PGCONTROL . ')$'))) - { - return true; - } - - return false; -} - -push @EXPORT, qw(isChecksumPage); - -1; diff --git a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm deleted file mode 100644 index 911c42a44..000000000 --- a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm +++ /dev/null @@ -1,747 +0,0 @@ -#################################################################################################################################### -# Test All Commands On PostgreSQL Clusters -#################################################################################################################################### -package pgBackRestTest::Module::Real::RealAllTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Storage; -use pgBackRestTest::Common::StoragePosix; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Backup advisory lock -#################################################################################################################################### -use constant DB_BACKUP_ADVISORY_LOCK => '12340078987004321'; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - foreach my $rhRun - ( - {pg => '9.4', dst => 'db-standby', tls => 0, stg => POSIX, enc => 1, cmp => LZ4, rt => 1, bnd => 1, bi => 0}, - {pg => '9.5', dst => 'backup', tls => 1, stg => GCS, enc => 0, cmp => BZ2, rt => 1, bnd => 0, bi => 1}, - {pg => '9.6', dst => 'backup', tls => 0, stg => POSIX, enc => 0, cmp => NONE, rt => 2, bnd => 1, bi => 1}, - {pg => '10', dst => 'sftp-srvr', tls => 0, stg => SFTP, enc => 1, cmp => GZ, rt => 1, bnd => 1, bi => 0}, - {pg => '11', dst => 'backup', tls => 1, stg => AZURE, enc => 0, cmp => ZST, rt => 2, bnd => 0, bi => 0}, - {pg => '12', dst => 'backup', tls => 0, stg => S3, enc => 1, cmp => LZ4, rt => 1, bnd => 0, bi => 1}, - {pg => '13', dst => 'db-standby', tls => 1, stg => GCS, enc => 0, cmp => ZST, rt => 1, bnd => 1, bi => 1}, - {pg => '14', dst => 'sftp-srvr', tls => 0, stg => SFTP, enc => 0, cmp => LZ4, rt => 1, bnd => 1, bi => 0}, - {pg => '15', dst => 'db-standby', tls => 0, stg => AZURE, enc => 0, cmp => NONE, rt => 2, bnd => 1, bi => 1}, - {pg => '16', dst => 'backup', tls => 0, stg => S3, enc => 1, cmp => NONE, rt => 1, bnd => 0, bi => 0}, - ) - { - # Only run tests for this pg version - next if ($rhRun->{pg} ne $self->pgVersion()); - - # Get run parameters - my $bHostBackup = $rhRun->{dst} eq HOST_BACKUP ? true : false; - my $bTls = $rhRun->{tls}; - my $strBackupDestination = $rhRun->{dst}; - my $strStorage = $rhRun->{stg}; - my $bRepoEncrypt = $rhRun->{enc}; - my $strCompressType = $rhRun->{cmp}; - my $iRepoTotal = $rhRun->{rt}; - my $bBundle = $rhRun->{bnd}; - my $bBlockIncr = $rhRun->{bi}; - - # Some tests are not version specific so only run them on a single version of PostgreSQL - my $bNonVersionSpecific = $self->pgVersion() eq PG_VERSION_96; - - # Increment the run, log, and decide whether this unit test should be run - next if !$self->begin( - "bkp ${bHostBackup}, tls ${bTls}, dst ${strBackupDestination}, cmp ${strCompressType}, storage ${strStorage}" . - ", enc ${bRepoEncrypt}, bi ${bBlockIncr}"); - - # Create hosts, file object, and config - my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup( - false, - {bHostBackup => $bHostBackup, bStandby => true, bTls => $bTls, strBackupDestination => $strBackupDestination, - strCompressType => $strCompressType, bArchiveAsync => false, strStorage => $strStorage, - bRepoEncrypt => $bRepoEncrypt, iRepoTotal => $iRepoTotal, bBundle => $bBundle, bBlockIncr => $bBlockIncr}); - - # Some commands will fail because of the bogus host created when a standby is present. These options reset the bogus host - # so it won't interfere with commands that won't tolerate a connection failure. - my $strBogusReset = $oHostBackup->bogusHost() ? - ' --reset-pg2-host --reset-pg2-host-type --reset-pg2-host-cmd --reset-pg2-host-config --reset-pg2-host-user' . - ' --reset-pg2-path' : - ''; - - # If S3 set process max to 2. This seems like the best place for parallel testing since it will help speed S3 processing - # without slowing down the other tests too much. - if ($strStorage eq S3) - { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); - $oHostDbPrimary->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); - } - - $oHostDbPrimary->clusterCreate(); - - # Create the stanza - $oHostDbPrimary->stanzaCreate('main create stanza info files'); - - # Get passphrase to access the Manifest file from backup.info - returns undefined if repo not encrypted - my $strCipherPass = - (new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()))->cipherPassSub(); - - # Create a manifest with the pg version to get version-specific paths - my $oManifest = new pgBackRestTest::Env::Manifest(BOGUS, {bLoad => false, strDbVersion => $self->pgVersion(), - iDbCatalogVersion => $self->dbCatalogVersion($self->pgVersion()), - strCipherPass => $strCipherPass, strCipherPassSub => $bRepoEncrypt ? ENCRYPTION_KEY_BACKUPSET : undef}); - - # Static backup parameters - my $fTestDelay = 1; - - # Restore test string - my $strDefaultMessage = 'default'; - my $strFullMessage = 'full'; - my $strStandbyMessage = 'standby'; - my $strIncrMessage = 'incr'; - my $strTimeMessage = 'time'; - my $strXidMessage = 'xid'; - my $strNameMessage = 'name'; - my $strTimelineMessage = 'timeline'; - - # Create two new databases - $oHostDbPrimary->sqlExecute('create database test1', {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute('create database test2', {bAutoCommit => true}); - - # ??? Removed temporarily until manifest build can be brought back into the check command - # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check - # -------------------------------------------------------------------------------------------------------------------------- - # my $strDir = $oHostDbPrimary->dbBasePath() . '/rootreaddir'; - # executeTest('sudo mkdir ' . $strDir); - # executeTest("sudo chown root:root ${strDir}"); - # executeTest("sudo chmod 400 ${strDir}"); - # - # $strComment = 'confirm primary manifest->build executed'; - # $oHostDbPrimary->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN}); - # executeTest("sudo rmdir ${strDir}"); - - # -------------------------------------------------------------------------------------------------------------------------- - my $strComment = 'verify check command runs successfully'; - - $oHostDbPrimary->check($strComment, {iTimeout => 10, bStanza => false}); - - # Also run check on the backup host when present - if ($bHostBackup) - { - $oHostBackup->check($strComment, {iTimeout => 10, strOptionalParam => $strBogusReset}); - } - - # Restart the cluster ignoring any errors in the postgresql log - $oHostDbPrimary->clusterRestart({bIgnoreLogError => true}); - - # Full backup - #--------------------------------------------------------------------------------------------------------------------------- - # Create the table where test messages will be stored - $oHostDbPrimary->sqlExecute("create table test (message text not null)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into test values ('$strDefaultMessage')"); - - # Acquire the backup advisory lock so it looks like a backup is running - if (!$oHostDbPrimary->sqlSelectOne('select pg_try_advisory_lock(' . DB_BACKUP_ADVISORY_LOCK . ')')) - { - confess 'unable to acquire advisory lock for testing'; - } - - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'fail on backup lock exists', {iExpectedExitStatus => ERROR_LOCK_ACQUIRE}); - - # Release the backup advisory lock so the next backup will succeed - if (!$oHostDbPrimary->sqlSelectOne('select pg_advisory_unlock(' . DB_BACKUP_ADVISORY_LOCK . ')')) - { - confess 'unable to release advisory lock'; - } - - $oHostDbPrimary->sqlExecute("update test set message = '$strFullMessage'"); - - # Required to set hint bits to be sent to the standby to make the heap match on both sides - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strFullMessage); - - # Backup to repo1 - my $strFullBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'repo1', - {strOptionalParam => ' --buffer-size=16384'}); - - # Backup to repo2 if it exists - if ($iRepoTotal == 2) - { - $oHostBackup->backup(CFGOPTVAL_BACKUP_TYPE_FULL, 'repo2', {iRepo => 2}); - } - - # Make a new backup with expire-auto disabled then run the expire command and compare backup numbers to ensure that expire - # was really disabled. This test is not version specific so is run on only one version. - #--------------------------------------------------------------------------------------------------------------------------- - if ($bNonVersionSpecific) - { - my $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - push(my @backupLst1, $oBackupInfo->list()); - - $strFullBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'with disabled expire-auto', - {strOptionalParam => ' --repo1-retention-full='.scalar(@backupLst1). ' --no-expire-auto'}); - - $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - push(my @backupLst2, $oBackupInfo->list()); - - &log(INFO, " run the expire command"); - $oHostBackup->expire({iRetentionFull => scalar(@backupLst1)}); - $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - push(my @backupLst3, $oBackupInfo->list()); - - unless (scalar(@backupLst2) == scalar(@backupLst1) + 1 && scalar(@backupLst1) == scalar(@backupLst3)) - { - confess "expire-auto option didn't work as expected"; - } - } - - # Enabled async archiving - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'archive-async' => 'y'}}); - - # Kick out a bunch of archive logs to exercise async archiving. Only do this when compressed and remote to slow it down - # enough to make it evident that the async process is working. - if ($strCompressType ne NONE && $strBackupDestination eq HOST_BACKUP) - { - &log(INFO, ' multiple wal switches to exercise async archiving'); - $oHostDbPrimary->sqlExecute("create table wal_activity (id int)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (1)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (2)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (3)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (4)"); - $oHostDbPrimary->sqlWalRotate(); - } - - # Setup replica - #--------------------------------------------------------------------------------------------------------------------------- - my %oRemapHash; - $oRemapHash{&MANIFEST_TARGET_PGDATA} = $oHostDbStandby->dbBasePath(); - - $oHostDbStandby->linkRemap($oManifest->walPath(), $oHostDbStandby->dbPath() . '/' . $oManifest->walPath()); - - $oHostDbStandby->restore( - 'restore backup on replica', 'latest', - {rhRemapHash => \%oRemapHash, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY, - strOptionalParam => - ' --recovery-option="primary_conninfo=host=' . HOST_DB_PRIMARY . - ' port=' . $oHostDbPrimary->pgPort() . ' user=replicator"'}); - - $oHostDbStandby->clusterStart({bHotStandby => true}); - - # Make sure streaming replication is on - $oHostDbPrimary->sqlSelectOneTest( - "select client_addr || '-' || state from pg_stat_replication", $oHostDbStandby->ipGet() . '/32-streaming'); - - # Check that the cluster was restored properly - $oHostDbStandby->sqlSelectOneTest('select message from test', $strFullMessage); - - # Update message for standby - $oHostDbPrimary->sqlExecute("update test set message = '$strStandbyMessage'"); - - if (!$bTls) - { - # If there is only a primary and a replica and the replica is the backup destination, then if pg2-host and - # pg256-host are BOGUS, confirm failure to reach the primary - if (!$bHostBackup && $strBackupDestination eq HOST_DB_STANDBY) - { - my $strStandbyBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to reach primary', - {bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg256-host=' . BOGUS}); - } - else - { - my $strStandbyBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to access at least one standby', - {bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg256-host=' . BOGUS}); - } - } - - my $strStandbyBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby', - {bStandby => true, iExpectedExitStatus => undef, strOptionalParam => '--repo1-retention-full=1'}); - - $strFullBackup = $strStandbyBackup; - - # ??? Removed temporarily until manifest build can be brought back into the check command - # # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check - # my $strDir = $oHostDbStandby->dbBasePath() . '/rootreaddir'; - # executeTest('sudo mkdir ' . $strDir); - # executeTest("sudo chown root:root ${strDir}"); - # executeTest("sudo chmod 400 ${strDir}"); - # - # my $strComment = 'confirm standby manifest->build executed'; - # - # # If there is an invalid host, the final error returned from check will be the inability to resolve the name which is - # # an open error instead of a read error - # if (!$oHostDbStandby->bogusHost()) - # { - # $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN}); - # } - # else - # { - # $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_READ}); - # } - # - # # Remove the directory in pg_data location that is only readable by root - # executeTest("sudo rmdir ${strDir}"); - - # Confirm the check command runs without error on a standby (when a bogus host is not configured) - $oHostDbStandby->check('verify check command on standby', {strOptionalParam => $strBogusReset}); - - # Shutdown the standby before creating tablespaces (this will error since paths are different) - $oHostDbStandby->clusterStop({bIgnoreLogError => true}); - - my $strAdhocBackup; - - # Execute stop and make sure the backup fails - #--------------------------------------------------------------------------------------------------------------------------- - # Restart the cluster to check for any errors before continuing since the stop tests will definitely create errors and the - # logs will need to be deleted to avoid causing issues further down the line. This test is not version specific so is run on - # only one version. - if ($bNonVersionSpecific) - { - confess "test must be performed on posix storage" if $strStorage ne POSIX; - - $oHostDbPrimary->clusterRestart(); - - # Add backup for adhoc expire - $strAdhocBackup = $oHostBackup->backup(CFGOPTVAL_BACKUP_TYPE_DIFF, 'backup for adhoc expire'); - - $oHostDbPrimary->stop(); - - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'attempt backup when stopped', - {iExpectedExitStatus => $oHostBackup == $oHostDbPrimary ? ERROR_STOP : ERROR_DB_CONNECT}); - - $oHostDbPrimary->start(); - } - - # Setup the time targets - #--------------------------------------------------------------------------------------------------------------------------- - # If the tests are running quickly then the time target might end up the same as the end time of the prior full backup. That - # means restore auto-select will not pick it as a candidate and restore the last backup instead causing the restore compare - # to fail. So, sleep one second. - sleep(1); - - $oHostDbPrimary->sqlExecute("update test set message = '$strTimeMessage'"); - $oHostDbPrimary->sqlWalRotate(); - my $strTimeTarget = $oHostDbPrimary->sqlSelectOne("select current_timestamp"); - &log(INFO, " time target is ${strTimeTarget}"); - - # Incr backup - fail on archive_mode=always when version >= 9.5 - #--------------------------------------------------------------------------------------------------------------------------- - if ($oHostDbPrimary->pgVersion() >= PG_VERSION_95) - { - # Set archive_mode=always - $oHostDbPrimary->clusterRestart({bArchiveAlways => true}); - - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on archive_mode=always', {iExpectedExitStatus => ERROR_FEATURE_NOT_SUPPORTED}); - - # Reset the cluster to a normal state so the next test will work - $oHostDbPrimary->clusterRestart(); - } - - # Incr backup - #--------------------------------------------------------------------------------------------------------------------------- - # Create a tablespace directory - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(1), {strMode => '0700', bCreateParent => true}); - - # Also create it on the standby so replay won't fail - if (defined($oHostDbStandby)) - { - storageTest()->pathCreate($oHostDbStandby->tablespacePath(1), {strMode => '0700', bCreateParent => true}); - } - - $oHostDbPrimary->sqlExecute( - "create tablespace ts1 location '" . $oHostDbPrimary->tablespacePath(1) . "'", {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute("alter table test set tablespace ts1"); - - # Create a table in the tablespace that will not be modified again to be sure it does get full page writes in the WAL later - $oHostDbPrimary->sqlExecute("create table test_exists (id int) tablespace ts1", {bCommit => true, bCheckPoint => true}); - - # Create a table in the tablespace - $oHostDbPrimary->sqlExecute("create table test_remove (id int)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("update test set message = '$strDefaultMessage'"); - $oHostDbPrimary->sqlWalRotate(); - - # Create a database in the tablespace and a table to check - $oHostDbPrimary->sqlExecute("create database test3 with tablespace ts1", {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute( - 'create table test3_exists (id int);' . - 'insert into test3_exists values (1);', - {strDb => 'test3', bAutoCommit => true}); - - # Create a table in test1 to check - test1 will not be restored - $oHostDbPrimary->sqlExecute( - 'create table test1_zeroed (id int);' . - 'insert into test1_zeroed values (1);', - {strDb => 'test1', bAutoCommit => true}); - - # Start a backup so the next backup has to restart it. This test is not required for PostgreSQL >= 9.6 since backups are run - # in non-exclusive mode. - if ($oHostDbPrimary->pgVersion() < PG_VERSION_96) - { - $oHostDbPrimary->sqlSelectOne("select pg_start_backup('test backup that will cause an error', true)"); - - # Verify that an error is returned if the backup is already running - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on backup already running', {iExpectedExitStatus => ERROR_DB_QUERY}); - - # Restart the cluster ignoring any errors in the postgresql log - $oHostDbPrimary->clusterRestart({bIgnoreLogError => true}); - - # Start a new backup to make the next test restarts it - $oHostDbPrimary->sqlSelectOne("select pg_start_backup('test backup that will be restarted', true)"); - } - - if (defined($strAdhocBackup)) - { - # Adhoc expire the latest backup - no other tests should be affected - $oHostBackup->expire({strOptionalParam => '--set=' . $strAdhocBackup}); - } - - # Drop a table - $oHostDbPrimary->sqlExecute('drop table test_remove'); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("update test set message = '$strIncrMessage'", {bCommit => true}); - - # Exercise --delta checksum option - my $strIncrBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'delta', - {strOptionalParam => '--stop-auto --buffer-size=32768 --delta', iRepo => $iRepoTotal}); - - # Ensure the check command runs properly with a tablespace - $oHostBackup->check( 'check command with tablespace', {iTimeout => 10, strOptionalParam => $strBogusReset}); - - # Setup the xid target - #--------------------------------------------------------------------------------------------------------------------------- - my $strXidTarget = undef; - - $oHostDbPrimary->sqlExecute("update test set message = '$strXidMessage'", {bCommit => false}); - $oHostDbPrimary->sqlWalRotate(); - $strXidTarget = $oHostDbPrimary->sqlSelectOne("select txid_current()"); - $oHostDbPrimary->sqlCommit(); - &log(INFO, " xid target is ${strXidTarget}"); - - # Setup the name target - #--------------------------------------------------------------------------------------------------------------------------- - my $strNameTarget = 'backrest'; - - $oHostDbPrimary->sqlExecute("update test set message = '$strNameMessage'", {bCommit => true}); - $oHostDbPrimary->sqlWalRotate(); - - $oHostDbPrimary->sqlExecute("select pg_create_restore_point('${strNameTarget}')"); - - &log(INFO, " name target is ${strNameTarget}"); - - # Create a table and data in database test2 - #--------------------------------------------------------------------------------------------------------------------------- - # Initialize variables for SHA1 and path of the pg_filenode.map for the database that will not be restored - my $strDb1TablePath; - my $strDb1TableSha1; - - $oHostDbPrimary->sqlExecute( - 'create table test (id int);' . - 'insert into test values (1);' . - 'create table test_ts1 (id int) tablespace ts1;' . - 'insert into test_ts1 values (2);', - {strDb => 'test2', bAutoCommit => true}); - - $oHostDbPrimary->sqlWalRotate(); - - # Get the SHA1 and path of the table for the database that will not be restored - $strDb1TablePath = $oHostDbPrimary->dbBasePath(). "/base/" . - $oHostDbPrimary->sqlSelectOne("select oid from pg_database where datname='test1'") . "/" . - $oHostDbPrimary->sqlSelectOne("select relfilenode from pg_class where relname='test1_zeroed'", {strDb => 'test1'}); - $strDb1TableSha1 = storageTest()->hashSize($strDb1TablePath); - - # Restore (type = default) - #--------------------------------------------------------------------------------------------------------------------------- - # Expect failure because pg (appears to be) running - $oHostDbPrimary->restore('pg running', 'latest', {iExpectedExitStatus => ERROR_PG_RUNNING}); - - $oHostDbPrimary->clusterStop(); - - # Expect failure because db path is not empty - $oHostDbPrimary->restore('path not empty', 'latest', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY}); - - # Drop and recreate db path - testPathRemove($oHostDbPrimary->dbBasePath()); - storageTest()->pathCreate($oHostDbPrimary->dbBasePath(), {strMode => '0700'}); - testPathRemove($oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath()); - storageTest()->pathCreate($oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath(), {strMode => '0700'}); - testPathRemove($oHostDbPrimary->tablespacePath(1)); - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(1), {strMode => '0700'}); - - # Now the restore should work - $oHostDbPrimary->restore( - undef, 'latest', - {strOptionalParam => ' --db-include=test2 --db-include=test3 --buffer-size=16384', iRepo => $iRepoTotal}); - - # Test that the first database has not been restored since --db-include did not include test1 - my ($strSHA1, $lSize) = storageTest()->hashSize($strDb1TablePath); - - # Create a zeroed sparse file in the test directory that is the same size as the filenode.map. We need to use the posix - # driver directly to do this because handles cannot be passed back from the C code. - my $oStorageTrunc = new pgBackRestTest::Common::Storage($self->testPath(), new pgBackRestTest::Common::StoragePosix()); - - my $strTestTable = $self->testPath() . "/testtable"; - my $oDestinationFileIo = $oStorageTrunc->openWrite($strTestTable); - $oDestinationFileIo->open(); - - # Truncate to the original size which will create a sparse file. - if (!truncate($oDestinationFileIo->handle(), $lSize)) - { - confess "unable to truncate '$strTestTable' with handle " . $oDestinationFileIo->handle(); - } - $oDestinationFileIo->close(); - - # Confirm the test filenode.map and the database test1 filenode.map are zeroed - my ($strSHA1Test, $lSizeTest) = storageTest()->hashSize($strTestTable); - $self->testResult(sub {($strSHA1Test eq $strSHA1) && ($lSizeTest == $lSize) && ($strSHA1 ne $strDb1TableSha1)}, - true, 'database test1 not restored'); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strNameMessage); - - # Once the cluster is back online, make sure the database & table in the tablespace exists properly - $oHostDbPrimary->sqlSelectOneTest('select id from test_ts1', 2, {strDb => 'test2'}); - $oHostDbPrimary->sqlDisconnect({strDb => 'test2'}); - - $oHostDbPrimary->sqlSelectOneTest('select id from test3_exists', 1, {strDb => 'test3'}); - $oHostDbPrimary->sqlDisconnect({strDb => 'test3'}); - - # The tablespace path should exist and have files in it - my $strTablespacePath = $oHostDbPrimary->tablespacePath(1); - - # Backup info will have the catalog number - my $oBackupInfo = new pgBackRestDoc::Common::Ini( - storageRepo(), $oHostBackup->repoBackupPath(FILE_BACKUP_INFO), - {bLoad => false, strContent => ${storageRepo()->get($oHostBackup->repoBackupPath(FILE_BACKUP_INFO))}}); - - # Construct the special path - $strTablespacePath .= - '/PG_' . $oHostDbPrimary->pgVersion() . qw{_} . $oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG); - - # Check that path exists - if (!storageTest()->pathExists($strTablespacePath)) - { - confess &log(ASSERT, "unable to find tablespace path '${strTablespacePath}'"); - } - - # Make sure there are some files in the tablespace path - if (grep(!/^PG\_VERSION$/i, storageTest()->list($strTablespacePath)) == 0) - { - confess &log(ASSERT, "no files found in tablespace path '${strTablespacePath}'"); - } - - # This table should exist to prove that the tablespace was restored. It has not been updated since it was created so it - # should not be created by any full page writes. Once it is verified to exist it can be dropped. - $oHostDbPrimary->sqlSelectOneTest("select count(*) from test_exists", 0); - $oHostDbPrimary->sqlExecute('drop table test_exists'); - - # Now it should be OK to drop database test2 and test3 - $oHostDbPrimary->sqlExecute('drop database test2', {bAutoCommit => true}); - - # The test table lives in ts1 so it needs to be moved or dropped - $oHostDbPrimary->sqlExecute('alter table test set tablespace pg_default'); - - # And drop the tablespace - $oHostDbPrimary->sqlExecute('drop database test3', {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute("drop tablespace ts1", {bAutoCommit => true}); - - # Restore (restore type = immediate, inclusive) - #--------------------------------------------------------------------------------------------------------------------------- - if ($oHostDbPrimary->pgVersion() >= PG_VERSION_94) - { - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_IMMEDIATE); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, $strFullBackup, {bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_IMMEDIATE, strTargetAction => 'promote'}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', ($strStandbyMessage)); - } - - # Restore (restore type = xid, inclusive) - #--------------------------------------------------------------------------------------------------------------------------- - my $strRecoveryFile = undef; - - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_XID); - - $oHostDbPrimary->clusterStop(); - - executeTest('rm -rf ' . $oHostDbPrimary->dbBasePath() . "/*"); - executeTest('rm -rf ' . $oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath() . '/*'); - - $oHostDbPrimary->restore( - undef, $strIncrBackup, - {bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_XID, strTarget => $strXidTarget, strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef, - strOptionalParam => '--tablespace-map-all=../../tablespace', bTablespace => false, - iRepo => $iRepoTotal}); - - # Save recovery file to test so we can use it in the next test - $strRecoveryFile = $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'postgresql.auto.conf' : DB_FILE_RECOVERYCONF; - - storageTest()->copy( - $oHostDbPrimary->dbBasePath() . qw{/} . $strRecoveryFile, $self->testPath() . qw{/} . $strRecoveryFile); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strXidMessage); - - $oHostDbPrimary->sqlExecute("update test set message = '$strTimelineMessage'"); - - # Restore (restore type = preserve, inclusive) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_PRESERVE); - - $oHostDbPrimary->clusterStop(); - - executeTest('rm -rf ' . $oHostDbPrimary->dbBasePath() . "/*"); - executeTest('rm -rf ' . $oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath() . '/*'); - executeTest('rm -rf ' . $oHostDbPrimary->tablespacePath(1) . "/*"); - - # Restore recovery file that was saved in last test - storageTest()->move($self->testPath . "/${strRecoveryFile}", $oHostDbPrimary->dbBasePath() . "/${strRecoveryFile}"); - - # Also touch recovery.signal when required - if ($oHostDbPrimary->pgVersion() >= PG_VERSION_12) - { - storageTest()->put($oHostDbPrimary->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL); - } - - $oHostDbPrimary->restore(undef, 'latest', {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strXidMessage); - - $oHostDbPrimary->sqlExecute("update test set message = '$strTimelineMessage'"); - - # Restore (restore type = time, inclusive, automatically select backup) - there is no exclusive time test because I can't - # find a way to find the exact commit time of a transaction. - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_TIME); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, 'latest', - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_TIME, strTarget => $strTimeTarget, strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef, - strBackupExpected => $strFullBackup}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strTimeMessage); - - # Restore (restore type = xid, exclusive) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_XID); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, $strIncrBackup, - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_XID, strTarget => $strXidTarget, bTargetExclusive => true, - strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef, - iRepo => $iRepoTotal}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strIncrMessage); - - # Restore (restore type = name) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_NAME); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, 'latest', - {bDelta => true, bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_NAME, strTarget => $strNameTarget, - strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strNameMessage); - - # Restore (restore type = default, timeline = created by type = xid, inclusive recovery) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_DEFAULT); - - $oHostDbPrimary->clusterStop(); - - # The timeline to use for this test is subject to change based on tests being added or removed above. The best thing would - # be to automatically grab the timeline after the restore, but since this test has been stable for a long time it does not - # seem worth the effort to automate. - $oHostDbPrimary->restore( - undef, $strIncrBackup, - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY, strTargetTimeline => 4, iRepo => $iRepoTotal}); - - $oHostDbPrimary->clusterStart({bHotStandby => true}); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strTimelineMessage, {iTimeout => 120}); - - # Stop clusters to catch any errors in the postgres log - #--------------------------------------------------------------------------------------------------------------------------- - $oHostDbPrimary->clusterStop(); - - # Stanza-delete --force without access to pgbackrest on database host. This test is not version specific so is run on only - # one version. - #--------------------------------------------------------------------------------------------------------------------------- - if ($bNonVersionSpecific) - { - # Make sure this test has a backup host to work with - confess "test must run with backup dst = " . HOST_BACKUP if !$bHostBackup; - - $oHostDbPrimary->stop(); - $oHostBackup->stop({strStanza => $self->stanza}); - $oHostBackup->stanzaDelete( - "delete stanza with --force when pgbackrest on pg host not accessible", {strOptionalParam => ' --force'}); - $oHostDbPrimary->start(); - $oHostBackup->start(); - } - } -} - -1; diff --git a/test/src/build/config/config.yaml b/test/src/build/config/config.yaml index a12fc79cd..8006823c4 100644 --- a/test/src/build/config/config.yaml +++ b/test/src/build/config/config.yaml @@ -70,6 +70,12 @@ option: command: test: {} + pg-version: + type: string + default: invalid + command: + test: {} + profile: type: boolean default: false diff --git a/test/src/build/help/help.xml b/test/src/build/help/help.xml index 190179895..b9b93b01a 100644 --- a/test/src/build/help/help.xml +++ b/test/src/build/help/help.xml @@ -120,6 +120,16 @@ n + +