diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d702943c..224a837ed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -358,7 +358,7 @@ pgbackrest/test/test.pl --vm-build --vm=u20 pgbackrest-dev => Run a Specific Test Run ``` -pgbackrest/test/test.pl --vm=u20 --module=mock --test=archive --run=2 +pgbackrest/test/test.pl --vm=u20 --module=postgres --test=interface --run=2 --- output --- diff --git a/doc/xml/contributing.xml b/doc/xml/contributing.xml index fa0c05abf..3ac68d812 100644 --- a/doc/xml/contributing.xml +++ b/doc/xml/contributing.xml @@ -398,7 +398,7 @@ pgbackrest/doc/doc.pl --help Run a Specific Test Run - pgbackrest/test/test.pl {[dash]}-vm=u20 {[dash]}-module=mock {[dash]}-test=archive {[dash]}-run=2 + pgbackrest/test/test.pl {[dash]}-vm=u20 {[dash]}-module=postgres {[dash]}-test=interface {[dash]}-run=2 {[test-cmd-extra]} diff --git a/test/define.yaml b/test/define.yaml index 81acfc8f6..e47f1650f 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -1003,30 +1003,6 @@ unit: # ********************************************************************************************************************************** integration: - # ******************************************************************************************************************************** - - name: mock - - test: - # ---------------------------------------------------------------------------------------------------------------------------- - - name: stanza - total: 2 - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: expire - total: 2 - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: archive - total: 2 - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: archive-stop - total: 2 - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: all - total: 2 - # ******************************************************************************************************************************** - name: real db: true @@ -1048,11 +1024,6 @@ performance: - name: performance test: - # ---------------------------------------------------------------------------------------------------------------------------- - - name: archive-perl - total: 1 - binReq: true - # ---------------------------------------------------------------------------------------------------------------------------- - name: type total: 6 diff --git a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm deleted file mode 100644 index e9bfc4bf8..000000000 --- a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm +++ /dev/null @@ -1,1111 +0,0 @@ -#################################################################################################################################### -# Test All Commands on Mock Data -#################################################################################################################################### -package pgBackRestTest::Module::Mock::MockAllTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use File::Basename qw(basename dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostS3Test; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# backupRegExpGet -# -# Generate a regexp depending on the backups that need to be found. -#################################################################################################################################### -sub backupRegExpGet -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $bFull, - $bDifferential, - $bIncremental, - $bAnchor - ) = - logDebugParam - ( - __PACKAGE__ . '::backupRegExpGet', \@_, - {name => 'bFull', default => false}, - {name => 'bDifferential', default => false}, - {name => 'bIncremental', default => false}, - {name => 'bAnchor', default => true} - ); - - # One of the types must be selected - if (!($bFull || $bDifferential || $bIncremental)) - { - confess &log(ASSERT, 'at least one backup type must be selected'); - } - - # Standard regexp to match date and time formatting - my $strDateTimeRegExp = "[0-9]{8}\\-[0-9]{6}"; - # Start the expression with the anchor if requested, date/time regexp and full backup indicator - my $strRegExp = ($bAnchor ? '^' : '') . $strDateTimeRegExp . 'F'; - - # Add the diff and/or incr expressions if requested - if ($bDifferential || $bIncremental) - { - # If full requested then diff/incr is optional - if ($bFull) - { - $strRegExp .= "(\\_"; - } - # Else diff/incr is required - else - { - $strRegExp .= "\\_"; - } - - # Append date/time regexp for diff/incr - $strRegExp .= $strDateTimeRegExp; - - # Filter on both diff/incr - if ($bDifferential && $bIncremental) - { - $strRegExp .= '(D|I)'; - } - # Else just diff - elsif ($bDifferential) - { - $strRegExp .= 'D'; - } - # Else just incr - else - { - $strRegExp .= 'I'; - } - - # If full requested then diff/incr is optional - if ($bFull) - { - $strRegExp .= '){0,1}'; - } - } - - # Append the end anchor if requested - $strRegExp .= $bAnchor ? "\$" : ''; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strRegExp', value => $strRegExp} - ); -} - -#################################################################################################################################### -# backupLabel -# -# Get unique backup label. -#################################################################################################################################### -sub backupLabel -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorageRepo, - $strRepoBackupPath, - $strType, - $strBackupLabelLast, - $lTimestampStart - ) = - logDebugParam - ( - __PACKAGE__ . '::backupLabelFormat', \@_, - {name => 'oStorageRepo', trace => true}, - {name => 'strRepoBackupPath', trace => true}, - {name => 'strType', trace => true}, - {name => 'strBackupLabelLast', required => false, trace => true}, - {name => 'lTimestampStart', trace => true} - ); - - # Create backup label - my $strBackupLabel = backupLabelFormat($strType, $strBackupLabelLast, $lTimestampStart); - - # Make sure that the timestamp has not already been used by a prior backup. This is unlikely for online backups since there is - # already a wait after the manifest is built but it's still possible if the remote and local systems don't have synchronized - # clocks. In practice this is most useful for making offline testing faster since it allows the wait after manifest build to be - # skipped by dealing with any backup label collisions here. - if ($oStorageRepo->list( - $strRepoBackupPath, - {strExpression => - ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStart) . - ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? 'F' : '(D|I)$')}) || - $oStorageRepo->list( - "${strRepoBackupPath}/" . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTimestampStart), - {strExpression => - ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? '^' : '_') . timestampFileFormat(undef, $lTimestampStart) . - ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL ? 'F' : '(D|I)\.manifest\.gz$'), - bIgnoreMissing => true})) - { - waitRemainder(); - $strBackupLabel = backupLabelFormat($strType, $strBackupLabelLast, time()); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackupLabel', value => $strBackupLabel, trace => true} - ); -} - -#################################################################################################################################### -# Build PostgreSQL pages for testing -#################################################################################################################################### -sub pageBuild -{ - my $tPageSource = shift; - my $iBlockNo = shift; - my $iChecksum = shift; - my $iWalId = shift; - my $iWalOffset = shift; - - return - (defined($iWalId) ? pack('I', $iWalId) . pack('I', $iWalOffset) : substr($tPageSource, 0, 8)) . pack('S', $iChecksum) . - substr($tPageSource, 10); -} - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - foreach my $rhRun - ( - {vm => VM2, remote => false, tls => false, storage => GCS, encrypt => false, delta => true, compress => BZ2}, - {vm => VM2, remote => true, tls => false, storage => AZURE, encrypt => true, delta => false, compress => GZ}, - {vm => VM3, remote => false, tls => false, storage => POSIX, encrypt => false, delta => true, compress => ZST}, - {vm => VM3, remote => true, tls => true, storage => S3, encrypt => true, delta => false, compress => LZ4}, - {vm => VM4, remote => false, tls => false, storage => GCS, encrypt => false, delta => false, compress => GZ}, - {vm => VM4, remote => true, tls => true, storage => S3, encrypt => true, delta => true, compress => ZST}, - ) - { - # Only run tests for this vm - next if ($rhRun->{vm} ne vmTest($self->vm())); - - # Increment the run, log, and decide whether this unit test should be run - my $bRemote = $rhRun->{remote}; - my $bTls = $rhRun->{tls}; - my $strStorage = $rhRun->{storage}; - my $bEncrypt = $rhRun->{encrypt}; - my $bDeltaBackup = $rhRun->{delta}; - my $strCompressType = $rhRun->{compress}; - - # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin("rmt ${bRemote}, tls ${bTls}, storage ${strStorage}, enc ${bEncrypt}, delta ${bDeltaBackup}")) {next} - - # Create hosts, file object, and config - my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup( - true, - {bHostBackup => $bRemote, bTls => $bTls, strStorage => $strStorage, bRepoEncrypt => $bEncrypt, - strCompressType => NONE}); - - # If S3 set process max to 2. This seems like the best place for parallel testing since it will help speed S3 processing - # without slowing down the other tests too much. - if ($strStorage eq S3) - { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); - $oHostDbPrimary->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); - - # Reduce log level to warn because parallel tests do not create deterministic logs - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(WARN)}}); - $oHostDbPrimary->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(WARN)}}); - } - - # Get base time - my $lTime = time() - 10000; - - # Build the manifest - my %oManifest; - - $oManifest{&INI_SECTION_BACKREST}{&INI_KEY_VERSION} = PROJECT_VERSION; - $oManifest{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = REPOSITORY_FORMAT; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_CHECK} = JSON::PP::false; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_COPY} = JSON::PP::false; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY} = JSON::PP::false; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 16384; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE} = JSON::PP::true; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::false; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = NONE; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL} = 3; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK} = $bRemote ? 1 : 3; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = JSON::PP::false; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE} = JSON::PP::false; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = JSON::PP::false; - - if ($bEncrypt) - { - $oManifest{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS} = 'REPLACEME'; - } - - $oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG} = 201409291; - $oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CONTROL} = 942; - $oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_SYSTEM_ID} = 1000000000000000094; - $oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} = PG_VERSION_94; - $oManifest{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_ID} = 1; - - $oManifest{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} = - $oHostDbPrimary->dbBasePath(); - $oManifest{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_PATH; - - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA); - - $oHostDbPrimary->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_PGVERSION, PG_VERSION_94, - '184473f470864e067ee3a22e64b47b0a1c356f29', $lTime, undef, true); - - # Load sample page - my $tBasePage = ${storageTest()->get($self->dataPath() . '/page.bin')}; - my $iBasePageChecksum = 0x1B99; - - # Create base path - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base'); - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/1'); - - $oHostDbPrimary->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/1/12000', $tBasePage, - '22c98d248ff548311eda88559e4a8405ed77c003', $lTime); - $oHostDbPrimary->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/1/' . DB_FILE_PGVERSION, - PG_VERSION_94, '184473f470864e067ee3a22e64b47b0a1c356f29', $lTime, '660'); - - if (!$bRemote) - { - $oHostDbPrimary->executeSimple( - 'chown 7777 ' . $oHostDbPrimary->dbBasePath() . '/base/1/' . DB_FILE_PGVERSION, undef, 'root'); - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/1/' . DB_FILE_PGVERSION} - {&MANIFEST_SUBKEY_USER} = INI_FALSE; - } - - my $tPageInvalid17000 = $tBasePage . $tBasePage; - - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384'); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000', $tPageInvalid17000, - 'e0101dd8ffb910c9c202ca35b5f828bcb9697bed', $lTime, undef, undef, '1'); - $oHostDbPrimary->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/' . DB_FILE_PGVERSION, - PG_VERSION_94, '184473f470864e067ee3a22e64b47b0a1c356f29', $lTime); - - if (!$bRemote) - { - $oHostDbPrimary->executeSimple( - 'chown :7777 ' . $oHostDbPrimary->dbBasePath() . '/base/16384/' . DB_FILE_PGVERSION, undef, 'root'); - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/16384/' . DB_FILE_PGVERSION} - {&MANIFEST_SUBKEY_GROUP} = INI_FALSE; - } - - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768'); - - my $tPageValid = - pageBuild($tBasePage, 0, 0x1b99) . - pageBuild($tBasePage, 1, 0x1b9a) . - pageBuild($tBasePage, 2, 0x1b97) . - ("\0" x 8192); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768/33000', $tPageValid, '1d73a0052828531770e7c155aeb22338e017e196', - $lTime); - - my $iBlockOffset = 32767 * 131072; - - my $tPageValidSeg32767 = - pageBuild($tBasePage, $iBlockOffset + 0, 0xf7de) . - pageBuild($tBasePage, $iBlockOffset + 1, 0xf7df) . - ("\0" x 8192) . - ("\0" x 8192); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768/33000.32767', $tPageValidSeg32767, - '1d11c42e6080e805a7b12bf9f83f4def548d92ac', $lTime); - - my $tPageInvalid33001 = - pageBuild($tBasePage, 1, 0x1b9a) . - pageBuild($tBasePage, 1, 0x1b9a) . - pageBuild($tBasePage, 2, 0x1b97) . - pageBuild($tBasePage, 0, 0x1b99) . - pageBuild($tBasePage, 0, 0x1b99) . - pageBuild($tBasePage, 0, 0x1b99) . - pageBuild($tBasePage, 6, 0x1b9b) . - pageBuild($tBasePage, 0, 0x1b99); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768/33001', $tPageInvalid33001, - '6bf316f11d28c28914ea9be92c00de9bea6d9a6b', $lTime, undef, undef, '0, [3, 5], 7'); - - $oHostDbPrimary->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768/' . DB_FILE_PGVERSION, - PG_VERSION_94, '184473f470864e067ee3a22e64b47b0a1c356f29', $lTime); - - # Create global path - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'global'); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_PGCONTROL, '[replaceme]', - $self->archBits() == 32 ? '59331420592064afff045f352de3a1ce53b0dd21' : 'e8bd9ef18a0b58f89aed7e98a1c32119f03727ee', - $lTime - 100, undef, true); - - # Copy pg_control - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_94); - utime($lTime - 100, $lTime - 100, $oHostDbPrimary->dbBasePath() . '/' . DB_FILE_PGCONTROL) - or confess &log(ERROR, "unable to set time"); - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGCONTROL} - {&MANIFEST_SUBKEY_SIZE} = 8192; - - # Create tablespace path - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGTBLSPC); - - # Create paths/files to ignore - if (!$bRemote) - { - # Create temp dir and file that will be ignored - $oHostDbPrimary->dbPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/' . DB_FILE_PREFIX_TMP); - $oHostDbPrimary->dbFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'base/' . DB_FILE_PREFIX_TMP . '/' . DB_FILE_PREFIX_TMP . '.1', 'IGNORE'); - - # Create pg_dynshmem dir and file - only file will be ignored - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGDYNSHMEM); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGDYNSHMEM . '/anything.tmp', 'IGNORE'); - - # Create pg_notify dir and file - only file will be ignored - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGNOTIFY); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGNOTIFY . '/anything.tmp', 'IGNORE'); - - # Create pg_replslot dir and file - only file will be ignored - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGREPLSLOT); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGREPLSLOT . '/anything.tmp', 'IGNORE'); - - # Create pg_serial dir and file - only file will be ignored - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSERIAL); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSERIAL . '/anything.tmp', 'IGNORE'); - - # Create pg_snapshots dir and file - only file will be ignored - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSNAPSHOTS); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSNAPSHOTS . '/anything.tmp', 'IGNORE'); - - # Create pg_stat_tmp dir and file - only file will be ignored - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSTATTMP); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSTATTMP . '/anything.tmp', 'IGNORE'); - - # Create pg_subtrans dir and file - only file will be ignored - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSUBTRANS); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_PATH_PGSUBTRANS . '/anything.tmp', 'IGNORE'); - - # More files to ignore - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_POSTGRESQLAUTOCONFTMP, 'IGNORE'); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_POSTMTROPTS, 'IGNORE'); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_RECOVERYCONF, 'IGNORE'); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_RECOVERYDONE, 'IGNORE'); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'global/' . DB_FILE_PGINTERNALINIT, 'IGNORE'); - - # Unlog and temp files to ignore (unlog _init will NOT be ignored) - $oHostDbPrimary->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768/44000_init', $tPageValid, - '1d73a0052828531770e7c155aeb22338e017e196', $lTime); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768/44000', 'IGNORE'); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/32768/t333_44000', 'IGNORE'); - } - - # Help and Version. These have complete unit tests, so here just make sure there is output from the command line. - #--------------------------------------------------------------------------------------------------------------------------- - if ($self->runCurrent() == 1) - { - $oHostDbPrimary->executeSimple($self->backrestExe() . " version"); - $oHostDbPrimary->executeSimple($self->backrestExe() . " help version"); - } - - # Full backup - #--------------------------------------------------------------------------------------------------------------------------- - my $strType = CFGOPTVAL_BACKUP_TYPE_FULL; - my $strOptionalParam = '--manifest-save-threshold=3'; - my $strTestPoint; - - # Create the archive info file - $oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-online'}); - - # Create a link to postgresql.conf - storageTest()->pathCreate($oHostDbPrimary->dbPath() . '/pg_config', {strMode => '0700', bCreateParent => true}); - testFileCreate( - $oHostDbPrimary->dbPath() . '/pg_config/postgresql.conf', "listen_addresses = *\n", $lTime - 100); - testLinkCreate($oHostDbPrimary->dbPath() . '/pg_config/postgresql.conf.link', './postgresql.conf'); - - $oHostDbPrimary->manifestLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'postgresql.conf', - '../pg_config/postgresql.conf', true); - - # Create a link to pg_hba.conf - testFileCreate( - $oHostDbPrimary->dbPath() . '/pg_config/pg_hba.conf', "CONTENTS\n", $lTime - 100); - testLinkCreate($oHostDbPrimary->dbPath() . '/pg_config/pg_hba.conf.link', './pg_hba.conf'); - - $oHostDbPrimary->manifestLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_hba.conf', - '../pg_config/pg_hba.conf', true); - - # Create stat directory link and file - storageTest()->pathCreate($oHostDbPrimary->dbPath() . '/pg_stat', {strMode => '0700', bCreateParent => true}); - $oHostDbPrimary->manifestLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_stat', '../pg_stat'); - $oHostDbPrimary->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA . '/pg_stat', 'global.stat', 'stats', - 'e350d5ce0153f3e22d5db21cf2a4eff00f3ee877', $lTime - 100, undef, true); - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_clog'); - - # Create file with special characters - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'special-!_.*\'()&!@;:+,?', undef, undef, $lTime, undef, true); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = 1; - - my $strFullBackup = $oHostBackup->backup( - $strType, 'create pg_stat link, pg_clog dir', - {oExpectedManifest => \%oManifest, - strOptionalParam => $strOptionalParam . - # Pass ssh path to make sure it is used - ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . - # Pass bogus ssh port to make sure it is passed through the protocol layer (it won't be used) - ($bRemote ? ' --pg1-port=9999' : '') . - # Pass bogus socket path to make sure it is passed through the protocol layer (it won't be used) - ($bRemote ? ' --pg1-socket-path=/test_socket_path' : '') . - ' --buffer-size=16384 --checksum-page --process-max=1', - strRepoType => $strStorage eq POSIX ? CIFS : undef, strTest => $strTestPoint, fTestDelay => 0}); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE} = 65536; - - # Stop operations and make sure the correct error occurs - #--------------------------------------------------------------------------------------------------------------------------- - if ($strStorage eq POSIX) - { - # Test global stop - $oHostDbPrimary->stop({bForce => true}); - - $oHostBackup->backup( - $strType, 'global stop', - {oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_STOP}); - - # Test stanza stop - $oHostDbPrimary->stop({strStanza => $oHostDbPrimary->stanza()}); - - # This time a warning should be generated - $oHostDbPrimary->stop({strStanza => $oHostDbPrimary->stanza()}); - - $oHostBackup->backup( - $strType, 'stanza stop', - {oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_STOP}); - - $oHostDbPrimary->start({strStanza => $self->stanza()}); - $oHostDbPrimary->start(); - - # This time a warning should be generated - $oHostDbPrimary->start(); - } - - # Resume Full Backup - #--------------------------------------------------------------------------------------------------------------------------- - $strType = CFGOPTVAL_BACKUP_TYPE_FULL; - - # These files should never be backed up (this requires the next backup to do --force) - testFileCreate($oHostDbPrimary->dbBasePath() . '/' . DB_FILE_POSTMTRPID, 'JUNK'); - testFileCreate($oHostDbPrimary->dbBasePath() . '/' . DB_FILE_BACKUPLABELOLD, 'JUNK'); - testFileCreate($oHostDbPrimary->dbBasePath() . '/' . DB_FILE_RECOVERYCONF, 'JUNK'); - testFileCreate($oHostDbPrimary->dbBasePath() . '/' . DB_FILE_RECOVERYDONE, 'JUNK'); - - # Create files in root tblspc paths that should not be copied or deleted. - # This will be checked later after a --force restore. - my $strDoNotDeleteFile = $oHostDbPrimary->tablespacePath(1, 2) . '/donotdelete.txt'; - storageTest()->pathCreate(dirname($strDoNotDeleteFile), {strMode => '0700', bCreateParent => true}); - testFileCreate($strDoNotDeleteFile, 'DONOTDELETE-1-2'); - - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(1), {strMode => '0700', bCreateParent => true}); - testFileCreate($oHostDbPrimary->tablespacePath(1) . '/donotdelete.txt', 'DONOTDELETE-1'); - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(2), {strMode => '0700', bCreateParent => true}); - testFileCreate($oHostDbPrimary->tablespacePath(2) . '/donotdelete.txt', 'DONOTDELETE-2'); - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(2, 2), {strMode => '0700', bCreateParent => true}); - testFileCreate($oHostDbPrimary->tablespacePath(2, 2) . '/donotdelete.txt', 'DONOTDELETE-2-2'); - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(11), {strMode => '0700', bCreateParent => true}); - - # Resume by copying the valid full backup over the last aborted full backup if it exists, or by creating a new path - my $strResumeBackup = (storageRepo()->list( - $oHostBackup->repoBackupPath(), {strExpression => backupRegExpGet(true, true, true), strSortOrder => 'reverse'}))[0]; - my $strResumeLabel = $strResumeBackup ne $strFullBackup ? - $strResumeBackup : backupLabel(storageRepo(), $oHostBackup->repoBackupPath(), $strType, undef, time()); - my $strResumePath = $oHostBackup->repoBackupPath($strResumeLabel); - - forceStorageRemove(storageRepo(), $strResumePath, {bRecurse => true}); - forceStorageMove(storageRepo(), $oHostBackup->repoBackupPath($strFullBackup), $strResumePath); - - # Set ownership on base directory to bogus values - if (!$bRemote) - { - $oHostDbPrimary->executeSimple('chown 7777:7777 ' . $oHostDbPrimary->dbBasePath(), undef, 'root'); - $oHostDbPrimary->executeSimple('chmod 777 ' . $oHostDbPrimary->dbBasePath(), undef, 'root'); - $oManifest{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_USER} = INI_FALSE; - $oManifest{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_GROUP} = INI_FALSE; - $oManifest{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_MODE} = '0777'; - } - - $oHostBackup->manifestMunge( - basename($strResumePath), - {&MANIFEST_SECTION_BACKUP => {&MANIFEST_KEY_LABEL => $strResumeLabel}, - &MANIFEST_SECTION_TARGET_FILE => - {(&MANIFEST_TARGET_PGDATA . '/' . &DB_FILE_PGVERSION) => {&MANIFEST_SUBKEY_CHECKSUM => undef}}}, - false); - - # Remove the main manifest so the backup appears aborted - forceStorageRemove(storageRepo(), "${strResumePath}/" . FILE_MANIFEST); - - # Create a temp file in backup temp root to be sure it's deleted correctly - my $strTempFile = "${strResumePath}/file.tmp"; - - if ($strStorage ne POSIX) - { - storageRepo()->put($strTempFile, "TEMP"); - } - else - { - executeTest("touch ${strTempFile}", {bRemote => $bRemote}); - } - - # Add zero-sized file - $oHostDbPrimary->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'zero_from_start', undef, - undef, $lTime, undef, true); - - # Add files for testing backups when time changes but content doesn't, and when content changes but time and size don't - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'changetime.txt', 'SIZE', '88087292ed82e26f3eb824d0bffc05ccf7a30f8d', $lTime, - undef, true); - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'changecontent.txt', 'CONTENT', '238a131a3e8eb98d1fc5b27d882ca40b7618fd2a', $lTime, - undef, true); - - # Create files to be excluded with the --exclude option - $oHostBackup->configUpdate( - {(CFGDEF_SECTION_GLOBAL . ':backup') => - {'exclude' => ['postgresql.auto.conf', 'pg_log/', 'pg_log2', 'apipe']}}); - $oHostDbPrimary->dbLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'postgresql.auto.conf', - '../pg_config/postgresql.conf', true); - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_log'); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_log/logfile', 'IGNORE'); - $oHostDbPrimary->dbPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_log2'); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_log2/logfile', 'IGNORE'); - executeTest('mkfifo ' . $oHostDbPrimary->dbBasePath() . '/apipe'); - - $strFullBackup = $oHostBackup->backup( - $strType, 'resume', - {oExpectedManifest => \%oManifest, - strOptionalParam => '--force --checksum-page' . ($bDeltaBackup ? ' --delta' : '')}); - - # Remove pid so restore will succeed (the rest will be cleaned up by the delta) - storageTest->remove($oHostDbPrimary->dbBasePath() . '/' . DB_FILE_POSTMTRPID); - - # Restore - tests various mode, extra files/paths, missing files/paths - #--------------------------------------------------------------------------------------------------------------------------- - # Munge permissions/modes on files that will be fixed by the restore - if (!$bRemote) - { - $oHostDbPrimary->executeSimple( - "chown :7777 " . $oHostDbPrimary->dbBasePath() . '/base/1/' . DB_FILE_PGVERSION, undef, 'root'); - $oHostDbPrimary->executeSimple( - "chmod 600 " . $oHostDbPrimary->dbBasePath() . '/base/1/' . DB_FILE_PGVERSION, undef, 'root'); - } - - # Create a path and file that are not in the manifest - $oHostDbPrimary->dbPathCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'deleteme'); - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'deleteme/deleteme.txt', 'DELETEME'); - - # Change path mode - $oHostDbPrimary->dbPathMode(\%oManifest, MANIFEST_TARGET_PGDATA, 'base', '0777'); - - # Remove a path - $oHostDbPrimary->dbPathRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_clog'); - - # Remove a file - $oHostDbPrimary->dbFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000'); - - # Restore will set invalid user and group to root since the base path user/group are also invalid - if (!$bRemote) - { - $oManifest{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_USER} = 'root'; - $oManifest{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_GROUP} = 'root'; - - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/1/' . DB_FILE_PGVERSION} - {&MANIFEST_SUBKEY_USER} = 'root'; - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/16384/' . DB_FILE_PGVERSION} - {&MANIFEST_SUBKEY_GROUP} = 'root'; - } - - $oHostDbPrimary->restore( - 'add and delete files', $strFullBackup, - {rhExpectedManifest => \%oManifest, bDelta => true, strUser => !$bRemote ? 'root' : undef, - strOptionalParam => ' --link-all' . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '')}); - - # Remove excludes now that they just create noise in the log - $oHostBackup->configUpdate({(CFGDEF_SECTION_GLOBAL . ':backup') => {'exclude' => []}}); - - # Run again to fix permissions - if (!$bRemote) - { - # Reset the base path user and group for the next restore so files will be reset to the base path user/group - $oHostDbPrimary->executeSimple( - 'chown ' . TEST_USER . ':' . TEST_GROUP . ' ' . $oHostDbPrimary->dbBasePath(), undef, 'root'); - - $oHostBackup->manifestMunge( - $strFullBackup, - {&MANIFEST_SECTION_TARGET_PATH => - {&MANIFEST_TARGET_PGDATA => - {&MANIFEST_SUBKEY_USER => undef, &MANIFEST_SUBKEY_GROUP => undef}}}, - false); - - $oManifest{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_USER} = TEST_USER; - $oManifest{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_GROUP} = TEST_GROUP; - - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/1/' . DB_FILE_PGVERSION} - {&MANIFEST_SUBKEY_USER} = TEST_USER; - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/base/16384/' . DB_FILE_PGVERSION} - {&MANIFEST_SUBKEY_GROUP} = TEST_GROUP; - - $oHostDbPrimary->restore( - 'fix permissions', $strFullBackup, - {rhExpectedManifest => \%oManifest, bDelta => true, strUser => 'root', - strOptionalParam => ' --link-all --log-level-console=detail'}); - - # Fix and remove files that are now owned by root - $oHostBackup->executeSimple('chown -R ' . TEST_USER . ':' . TEST_GROUP . ' ' . $oHostBackup->logPath(), undef, 'root'); - $oHostDbPrimary->executeSimple('rm -rf ' . $oHostDbPrimary->lockPath() . '/*', undef, 'root'); - } - - # Change an existing link to the wrong directory - $oHostDbPrimary->dbFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_stat'); - $oHostDbPrimary->dbLinkCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_stat', '../wrong'); - - $oHostDbPrimary->restore( - 'fix broken symlink', $strFullBackup, - {rhExpectedManifest => \%oManifest, bDelta => true, - strOptionalParam => ' --link-all' . ($bRemote ? ' --compress-level-network=0' : '')}); - - # Restore links as directories - #--------------------------------------------------------------------------------------------------------------------------- - # Munge the user to make sure it gets reset on the next run - $oHostBackup->manifestMunge( - $strFullBackup, - {&MANIFEST_SECTION_TARGET_FILE => - {&MANIFEST_FILE_PGCONTROL => {&MANIFEST_SUBKEY_USER => 'bogus', &MANIFEST_SUBKEY_GROUP => 'bogus'}}}, - false); - - # Restore succeeds - $oHostDbPrimary->manifestLinkMap(\%oManifest, MANIFEST_TARGET_PGDATA . '/pg_stat'); - $oHostDbPrimary->manifestLinkMap(\%oManifest, MANIFEST_TARGET_PGDATA . '/postgresql.conf'); - $oHostDbPrimary->manifestLinkMap(\%oManifest, MANIFEST_TARGET_PGDATA . '/pg_hba.conf'); - - $oHostDbPrimary->restore( - 'restore links as directories', $strFullBackup, - {rhExpectedManifest => \%oManifest, bDelta => true, bForce => true}); - - # No longer need pg_hba.conf since it is no longer a link and doesn't provide additional coverage - $oHostDbPrimary->manifestFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'pg_hba.conf'); - - # Incr backup - #--------------------------------------------------------------------------------------------------------------------------- - $strType = CFGOPTVAL_BACKUP_TYPE_INCR; - $oHostDbPrimary->manifestReference(\%oManifest, $strFullBackup); - - # Fix mode on base path so defaults are simpler in backups - if (!$bRemote) - { - $oHostDbPrimary->executeSimple('chmod 700 ' . $oHostDbPrimary->dbBasePath(), undef, 'root'); - $oManifest{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_MODE} = '0700'; - } - - # Add tablespace 1 - $oHostDbPrimary->manifestTablespaceCreate(\%oManifest, 1); - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/1', '16384'); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGTBLSPC . '/1', '16384/tablespace1.txt', 'TBLSPCB', - '14c44cef6287269b08d41de489fd492bb9fc795d', $lTime - 100, undef, undef, false); - $oHostDbPrimary->manifestFileCreate(\%oManifest, MANIFEST_TARGET_PGDATA, 'badchecksum.txt', 'BADCHECKSUM', - 'f927212cd08d11a42a666b2f04235398e9ceeb51', $lTime, undef, true); - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'changesize.txt', 'SIZE', '88087292ed82e26f3eb824d0bffc05ccf7a30f8d', $lTime, - undef, true); - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'zerosize.txt', '', undef, $lTime - 100, undef, true); - - # Create temp dir and file that will be ignored - if (!$bRemote) - { - $oHostDbPrimary->dbPathCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/1', DB_FILE_PREFIX_TMP); - $oHostDbPrimary->dbFileCreate( - \%oManifest, MANIFEST_TARGET_PGTBLSPC . '/1', DB_FILE_PREFIX_TMP . '/' . DB_FILE_PREFIX_TMP . '.1', 'IGNORE'); - } - - my $strBackup = $oHostBackup->backup($strType, 'add tablespace 1', {oExpectedManifest => \%oManifest}); - - # Resume Incr Backup - #--------------------------------------------------------------------------------------------------------------------------- - $strType = CFGOPTVAL_BACKUP_TYPE_INCR; - - # Create resumable backup from last backup - $strResumeLabel = backupLabel(storageRepo(), $oHostBackup->repoBackupPath(), $strType, substr($strBackup, 0, 16), time()); - $strResumePath = $oHostBackup->repoBackupPath($strResumeLabel); - - forceStorageRemove(storageRepo(), $strResumePath); - forceStorageMove(storageRepo(), $oHostBackup->repoBackupPath($strBackup), $strResumePath); - - # Munge manifest so the resumed file in the repo appears to be bad - if ($bEncrypt || $bRemote) - { - $oHostBackup->manifestMunge( - basename($strResumePath), - {&MANIFEST_SECTION_TARGET_FILE => - {(&MANIFEST_TARGET_PGDATA . '/badchecksum.txt') => - {&MANIFEST_SUBKEY_CHECKSUM => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'}}}, - false); - } - # Change contents of resumed file without changing size so it will throw a nasty error about the repo having been corrupted - else - { - storageRepo()->put("${strResumePath}/pg_data/badchecksum.txt", 'BDDCHECKSUM'); - } - - # Write correct label into resumable manifest - $oHostBackup->manifestMunge( - basename($strResumePath), {&MANIFEST_SECTION_BACKUP => {&MANIFEST_KEY_LABEL => $strResumeLabel}},false); - - # Change contents/size of a db file to make sure it recopies (and does not resume) - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'changesize.txt', 'SIZE+MORE', '3905d5be2ec8d67f41435dab5e0dcda3ae47455d', $lTime, - undef, true); - - # Change contents/time of a db file to make sure it recopies (and does not resume) - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGTBLSPC . '/1', '16384/tablespace1.txt', 'TBLSPC1', - 'd85de07d6421d90aa9191c11c889bfde43680f0f', $lTime, undef, undef, false); - - # Remove the main manifest so the backup appears aborted - forceStorageRemove(storageRepo(), "${strResumePath}/" . FILE_MANIFEST); - - # Add tablespace 2 - $oHostDbPrimary->manifestTablespaceCreate(\%oManifest, 2); - $oHostDbPrimary->manifestPathCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768'); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/tablespace2.txt', 'TBLSPC2', - 'dc7f76e43c46101b47acc55ae4d593a9e6983578', $lTime, undef, undef, false); - - # Make sure pg_internal.init is ignored in tablespaces - $oHostDbPrimary->dbFileCreate(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/' . DB_FILE_PGINTERNALINIT, 'IGNORE'); - - # Also create tablespace 11 to be sure it does not conflict with path of tablespace 1 - $oHostDbPrimary->manifestTablespaceCreate(\%oManifest, 11); - - # Change only the time to be in the past on a valid file and update the timestamp in the expected manifest - utime($lTime - 100, $lTime - 100, $oHostDbPrimary->dbBasePath() . '/changetime.txt') - or confess &log(ERROR, "unable to set time for file ".$oHostDbPrimary->dbBasePath() . '/changetime.txt'); - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/changetime.txt'}{&MANIFEST_SUBKEY_TIMESTAMP} = $lTime - 100; - - # Change the content of the changecontent file to be the same size but leave the timestamp the same on the file - storageTest()->put($oHostDbPrimary->dbBasePath() . '/changecontent.txt', 'CHGCONT'); - utime($lTime, $lTime, $oHostDbPrimary->dbBasePath() . '/changecontent.txt') - or confess &log(ERROR, "unable to set time for file ".$oHostDbPrimary->dbBasePath() . '/changecontent.txt'); - - # The changecontent & changetime files have conditions that will force the delta option to be turned on which should result - # in the reference of changecontent to be removed but the reference to changetime to stay since the checksum wouldn't change - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/changecontent.txt'}{&MANIFEST_SUBKEY_CHECKSUM} = - "a094d94583e209556d03c3c5da33131a065f1689"; - delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/changecontent.txt'}{&MANIFEST_SUBKEY_REFERENCE}); - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/changetime.txt'}{&MANIFEST_SUBKEY_TIMESTAMP} = $lTime - 100; - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = 1; - - $strBackup = $oHostBackup->backup( - $strType, 'resume and add tablespace 2', - {oExpectedManifest => \%oManifest, - strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')}); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1; - - # Drop tablespace 11 - #--------------------------------------------------------------------------------------------------------------------------- - $strType = CFGOPTVAL_BACKUP_TYPE_DIFF; - - # Drop tablespace 11 - $oHostDbPrimary->manifestTablespaceDrop(\%oManifest, 11); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = 1; - - $strBackup = $oHostBackup->backup( - $strType, 'drop tablespace 11', - {oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')}); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1; - - # Restore - #--------------------------------------------------------------------------------------------------------------------------- - # Remap the base and tablespace paths - my %oRemapHash; - $oRemapHash{&MANIFEST_TARGET_PGDATA} = $oHostDbPrimary->dbBasePath(2); - storageTest()->pathCreate($oHostDbPrimary->dbBasePath(2), {strMode => '0700', bCreateParent => true}); - $oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/1'} = $oHostDbPrimary->tablespacePath(1, 2); - $oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/2'} = $oHostDbPrimary->tablespacePath(2, 2); - - $oHostDbPrimary->restore( - 'remap all paths', $strBackup, {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash}); - - # Restore (make sure file in root tablespace path is not deleted by --delta) - #--------------------------------------------------------------------------------------------------------------------------- - $oHostDbPrimary->restore( - 'ensure file in tblspc root remains after --delta', $strBackup, - {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true}); - - if (!-e $strDoNotDeleteFile) - { - confess "${strDoNotDeleteFile} was deleted by --delta"; - } - - # Incr Backup - #--------------------------------------------------------------------------------------------------------------------------- - $strType = CFGOPTVAL_BACKUP_TYPE_INCR; - $oHostDbPrimary->manifestReference(\%oManifest, $strBackup); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'base/base2.txt', 'BASE2', '09b5e31766be1dba1ec27de82f975c1b6eea2a92', $lTime); - - $oHostDbPrimary->manifestTablespaceDrop(\%oManifest, 1, 2); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/tablespace2b.txt', 'TBLSPC2B', - 'e324463005236d83e6e54795dbddd20a74533bf3', $lTime, undef, undef, false); - - # Munge the version to make sure it gets corrected on the next run - $oHostBackup->manifestMunge($strBackup, {&INI_SECTION_BACKREST => {&INI_KEY_VERSION => '0.00'}}, false); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = 1; - - $strBackup = $oHostBackup->backup( - $strType, 'add files and remove tablespace 2', - {oExpectedManifest => \%oManifest, strOptionalParam => '--process-max=1'}); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1; - - # Incr Backup - #--------------------------------------------------------------------------------------------------------------------------- - $strType = CFGOPTVAL_BACKUP_TYPE_INCR; - $oHostDbPrimary->manifestReference(\%oManifest, $strBackup); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000', 'BASEUPDT', '9a53d532e27785e681766c98516a5e93f096a501', - $lTime, undef, undef, false); - - # Perform the backup - $strBackup =$oHostBackup->backup($strType, 'update files', {oExpectedManifest => \%oManifest}); - - # Diff Backup - #--------------------------------------------------------------------------------------------------------------------------- - $strType = CFGOPTVAL_BACKUP_TYPE_DIFF; - $oHostDbPrimary->manifestReference(\%oManifest, $strFullBackup, true); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = 1; - - $strBackup = $oHostBackup->backup( - $strType, 'updates since last full', {oExpectedManifest => \%oManifest, - strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')}); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1; - - # Diff Backup with files removed - #--------------------------------------------------------------------------------------------------------------------------- - $oHostDbPrimary->manifestReference(\%oManifest, $strFullBackup, true); - - $strType = CFGOPTVAL_BACKUP_TYPE_DIFF; - - # Enable compression to ensure a warning is raised (reset when gz to avoid log churn since it is the default) - if ($strCompressType eq GZ) - { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'compress-type' => undef}}); - } - else - { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'compress-type' => $strCompressType}}); - } - - # Enable hardlinks (except for s3) to show they can be enabled after a full backup - if ($strStorage eq POSIX) - { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'repo1-hardlink' => 'y'}}); - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK} = JSON::PP::true; - $oHostBackup->{bHardLink} = true; - } - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = 1; - - $oHostDbPrimary->manifestFileRemove(\%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/tablespace2b.txt', true); - $oHostDbPrimary->manifestFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/base2.txt', true); - $oHostDbPrimary->manifestFileRemove(\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000'); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGTBLSPC . '/2', '32768/tablespace2c.txt', 'TBLSPCBIGGER', - 'dfcb8679956b734706cf87259d50c88f83e80e66', $lTime, undef, undef, false); - - $oHostBackup->backup( - $strType, 'remove files', - {oExpectedManifest => \%oManifest, - strOptionalParam => '--process-max=1' . ($bDeltaBackup ? ' --delta' : '')}); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX} = $strStorage eq S3 ? 2 : 1; - - # Full Backup - #--------------------------------------------------------------------------------------------------------------------------- - $strType = CFGOPTVAL_BACKUP_TYPE_FULL; - - # Now the compression changes will take effect - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS} = JSON::PP::true; - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE} = $strCompressType; - - $oHostDbPrimary->manifestReference(\%oManifest); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000', 'BASEUPDT2', '7579ada0808d7f98087a0a586d0df9de009cdc33', - $lTime, undef, undef, false); - - $oManifest{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE} = JSON::PP::false; - - $strFullBackup = $oHostBackup->backup( - $strType, 'update file', {oExpectedManifest => \%oManifest}); - - # Call expire - #--------------------------------------------------------------------------------------------------------------------------- - $oHostBackup->expire({iRetentionFull => 1}); - - # Diff Backup - #--------------------------------------------------------------------------------------------------------------------------- - $strType = CFGOPTVAL_BACKUP_TYPE_DIFF; - - $oHostDbPrimary->manifestReference(\%oManifest, $strFullBackup); - - $oHostDbPrimary->manifestFileCreate( - \%oManifest, MANIFEST_TARGET_PGDATA, 'base/base2.txt', 'BASE2UPDT', 'cafac3c59553f2cfde41ce2e62e7662295f108c0', - $lTime, undef, undef, false); - - # Munge the prior manifest so that option-checksum-page is missing to be sure the logic works for backups before page - # checksums were introduced - $oHostBackup->manifestMunge( - $strFullBackup, {&MANIFEST_SECTION_BACKUP_OPTION => {&MANIFEST_KEY_CHECKSUM_PAGE => undef}}, false); - - $strBackup = $oHostBackup->backup( - $strType, 'add file', {oExpectedManifest => \%oManifest, strOptionalParam => '--checksum-page'}); - - # Selective Restore - #--------------------------------------------------------------------------------------------------------------------------- - # Remove mapping for tablespace 1 - delete($oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/1'}); - - # Remove checksum to match zeroed files - delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/32768/33000'}{&MANIFEST_SUBKEY_CHECKSUM}); - delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/32768/33001'}{&MANIFEST_SUBKEY_CHECKSUM}); - delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_tblspc/2/PG_9.4_201409291/32768/tablespace2.txt'} - {&MANIFEST_SUBKEY_CHECKSUM}); - delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_tblspc/2/PG_9.4_201409291/32768/tablespace2c.txt'} - {&MANIFEST_SUBKEY_CHECKSUM}); - - $oHostDbPrimary->restore( - 'selective restore 16384', 'latest', - {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true, - strOptionalParam => '--db-include=16384'}); - - # Restore checksum values for next test - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/32768/33000'}{&MANIFEST_SUBKEY_CHECKSUM} = - '1d73a0052828531770e7c155aeb22338e017e196'; - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/32768/33001'}{&MANIFEST_SUBKEY_CHECKSUM} = - '6bf316f11d28c28914ea9be92c00de9bea6d9a6b'; - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_tblspc/2/PG_9.4_201409291/32768/tablespace2.txt'} - {&MANIFEST_SUBKEY_CHECKSUM} = 'dc7f76e43c46101b47acc55ae4d593a9e6983578'; - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_tblspc/2/PG_9.4_201409291/32768/tablespace2c.txt'} - {&MANIFEST_SUBKEY_CHECKSUM} = 'dfcb8679956b734706cf87259d50c88f83e80e66'; - - # Remove checksum to match zeroed file - delete($oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/16384/17000'}{&MANIFEST_SUBKEY_CHECKSUM}); - - $oHostDbPrimary->restore( - 'selective restore 32768', 'latest', - {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true, - strOptionalParam => '--db-include=32768'}); - - $oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/base/16384/17000'}{&MANIFEST_SUBKEY_CHECKSUM} = - '7579ada0808d7f98087a0a586d0df9de009cdc33'; - - $oHostDbPrimary->restore( - 'error on invalid id', 'latest', - {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true, - iExpectedExitStatus => ERROR_DB_MISSING, strOptionalParam => '--log-level-console=warn --db-include=7777'}); - - $oHostDbPrimary->restore( - 'error on system id', 'latest', - {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bDelta => true, - iExpectedExitStatus => ERROR_DB_INVALID, strOptionalParam => '--log-level-console=warn --db-include=1'}); - - # Compact Restore - #--------------------------------------------------------------------------------------------------------------------------- - executeTest('rm -rf ' . $oHostDbPrimary->dbBasePath(2) . "/*"); - - my $strDbPath = $oHostDbPrimary->dbBasePath(2) . '/base'; - storageTest()->pathCreate($strDbPath, {strMode => '0700'}); - - $oRemapHash{&MANIFEST_TARGET_PGDATA} = $strDbPath; - delete($oRemapHash{&MANIFEST_TARGET_PGTBLSPC . '/2'}); - - $oHostDbPrimary->restore( - 'no tablespace remap', 'latest', - {rhExpectedManifest => \%oManifest, rhRemapHash => \%oRemapHash, bTablespace => false, - strOptionalParam => '--tablespace-map-all=../../tablespace'}); - - $oManifest{&MANIFEST_SECTION_BACKUP_TARGET}{'pg_tblspc/2'}{&MANIFEST_SUBKEY_PATH} = '../../tablespace/ts2'; - $oManifest{&MANIFEST_SECTION_TARGET_LINK}{'pg_data/pg_tblspc/2'}{&MANIFEST_SUBKEY_DESTINATION} = '../../tablespace/ts2'; - - # Dump out history path at the end to verify all history files are being recorded. This test is only performed locally - # because for some reason sort order is different when this command is executed via ssh (even though the content of the - # directory is identical). - #--------------------------------------------------------------------------------------------------------------------------- - if (!$bRemote && $strStorage eq POSIX) - { - executeTest('ls -1Rtr ' . $oHostBackup->repoBackupPath(PATH_BACKUP_HISTORY), {bRemote => $bRemote}); - } - - # Test backup from standby warning that standby not configured so option reset - #--------------------------------------------------------------------------------------------------------------------------- - if (!defined($oHostDbStandby)) - { - $strBackup = $oHostBackup->backup( - $strType, 'option backup-standby reset - backup performed from primary', {oExpectedManifest => \%oManifest, - strOptionalParam => '--log-level-console=info --backup-standby'}); - } - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm deleted file mode 100644 index eb781db88..000000000 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm +++ /dev/null @@ -1,142 +0,0 @@ -#################################################################################################################################### -# Tests for archive-push command to make sure aync queue limits are implemented correctly -#################################################################################################################################### -package pgBackRestTest::Module::Mock::MockArchiveStopTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Manifest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - # Generate test WAL file - my $strWalTestFile = $self->testPath() . '/test-wal-' . PG_VERSION_94; - my $strWalHash = $self->walGenerateContentChecksum(PG_VERSION_94); - storageTest()->put($strWalTestFile, $self->walGenerateContent(PG_VERSION_94)); - - foreach my $rhRun - ( - {vm => VM2, remote => false, tls => false, storage => S3, encrypt => false, compress => NONE, error => 0}, - {vm => VM2, remote => true, tls => true, storage => POSIX, encrypt => true, compress => NONE, error => 0}, - {vm => VM3, remote => false, tls => false, storage => GCS, encrypt => true, compress => BZ2, error => 0}, - {vm => VM3, remote => true, tls => false, storage => AZURE, encrypt => false, compress => LZ4, error => 1}, - {vm => VM4, remote => false, tls => false, storage => S3, encrypt => true, compress => NONE, error => 0}, - {vm => VM4, remote => true, tls => true, storage => GCS, encrypt => false, compress => ZST, error => 0}, - ) - { - # Only run tests for this vm - next if ($rhRun->{vm} ne vmTest($self->vm())); - - # Increment the run, log, and decide whether this unit test should be run - my $bRemote = $rhRun->{remote}; - my $bTls = $rhRun->{tls}; - my $strStorage = $rhRun->{storage}; - my $bEncrypt = $rhRun->{encrypt}; - my $strCompressType = $rhRun->{compress}; - my $iError = $rhRun->{error}; - - # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin( - "rmt ${bRemote}, tls ${bTls}, cmp ${strCompressType}, error " . ($iError ? 'connect' : 'version') . - ", storage ${strStorage}, enc ${bEncrypt}")) {next} - - # Create hosts, file object, and config - my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup( - true, - {bHostBackup => $bRemote, bTls => $bTls, strCompressType => $strCompressType, bArchiveAsync => true, - strStorage => $strStorage, bRepoEncrypt => $bEncrypt}); - - # Create compression extension - my $strCompressExt = $strCompressType ne NONE ? ".${strCompressType}" : ''; - - # Create the wal path - my $strWalPath = $oHostDbPrimary->dbBasePath() . '/pg_xlog'; - storageTest()->pathCreate($strWalPath, {bCreateParent => true}); - - # Create the test path for pg_control and generate pg_control for stanza-create - storageTest()->pathCreate($oHostDbPrimary->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true}); - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_94); - - # Create the archive info file - $oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-online'}); - - # Push a WAL segment - &log(INFO, ' push first WAL'); - $oHostDbPrimary->archivePush($strWalPath, $strWalTestFile, 1); - - # Break the database version of the archive info file - if ($iError == 0) - { - $oHostBackup->infoMunge( - $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE), - {&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}, - &INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}}); - } - - # Push two more segments with errors to exceed archive-push-queue-max - &log(INFO, ' push second WAL'); - - $oHostDbPrimary->archivePush( - $strWalPath, $strWalTestFile, 2, ERROR_REPO_INVALID, undef, $iError ? '--repo1-host=bogus' : undef); - - &log(INFO, ' push third WAL'); - - $oHostDbPrimary->archivePush( - $strWalPath, $strWalTestFile, 3, ERROR_REPO_INVALID, undef, $iError ? '--repo1-host=bogus' : undef); - - # Now this segment will get dropped - &log(INFO, ' push fourth WAL'); - - $oHostDbPrimary->archivePush($strWalPath, $strWalTestFile, 4, undef, undef, '--repo1-host=bogus'); - - # Fix the database version - if ($iError == 0) - { - $oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE)); - } - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))}, - "000000010000000100000001-${strWalHash}${strCompressExt}", - 'segment 2-4 not pushed', {iWaitSeconds => 5}); - - #--------------------------------------------------------------------------------------------------------------------------- - $oHostDbPrimary->archivePush($strWalPath, $strWalTestFile, 5); - - $self->testResult( - sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))}, - "(000000010000000100000001-${strWalHash}${strCompressExt}, " . - "000000010000000100000005-${strWalHash}${strCompressExt})", - 'segment 5 is pushed', {iWaitSeconds => 5}); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm deleted file mode 100644 index 80d6db911..000000000 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm +++ /dev/null @@ -1,390 +0,0 @@ -#################################################################################################################################### -# Mock Archive Tests -#################################################################################################################################### -package pgBackRestTest::Module::Mock::MockArchiveTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::Manifest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# archiveCheck -# -# Check that a WAL segment is present in the repository. -#################################################################################################################################### -sub archiveCheck -{ - my $self = shift; - my $oHostBackup = shift; - my $strArchiveFile = shift; - my $strArchiveChecksum = shift; - my $strCompressType = shift; - my $strSpoolPath = shift; - - # Build the archive name to check for at the destination - my $strArchiveCheck = $oHostBackup->repoArchivePath( - PG_VERSION_94 . "-1/" . substr($strArchiveFile, 0, 16) . "/${strArchiveFile}-${strArchiveChecksum}"); - - if (defined($strCompressType)) - { - $strArchiveCheck .= ".${strCompressType}"; - } - - my $oWait = waitInit(5); - my $bFound = false; - - do - { - $bFound = storageRepo()->exists($strArchiveCheck); - } - while (!$bFound && waitMore($oWait)); - - if (!$bFound) - { - confess "unable to find ${strArchiveCheck}"; - } - - if (defined($strSpoolPath)) - { - storageTest()->remove("${strSpoolPath}/archive/" . $self->stanza() . "/out/${strArchiveFile}.ok"); - } -} - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - my $strArchiveChecksum = $self->walGenerateContentChecksum(PG_VERSION_94, {iSourceNo => 2}); - - foreach my $rhRun - ( - {vm => VM2, remote => false, tls => false, storage => S3, encrypt => true, compress => BZ2}, - {vm => VM2, remote => true, tls => false, storage => POSIX, encrypt => true, compress => BZ2}, - {vm => VM3, remote => false, tls => false, storage => POSIX, encrypt => true, compress => LZ4}, - {vm => VM3, remote => true, tls => true, storage => S3, encrypt => false, compress => ZST}, - {vm => VM4, remote => false, tls => false, storage => AZURE, encrypt => true, compress => ZST}, - {vm => VM4, remote => true, tls => true, storage => POSIX, encrypt => false, compress => GZ}, - ) - { - # Only run tests for this vm - next if ($rhRun->{vm} ne vmTest($self->vm())); - - # Increment the run, log, and decide whether this unit test should be run - my $bRemote = $rhRun->{remote}; - my $bTls = $rhRun->{tls}; - my $strStorage = $rhRun->{storage}; - my $bEncrypt = $rhRun->{encrypt}; - my $strCompressType = $rhRun->{compress}; - - if (!$self->begin("rmt ${bRemote}, tls ${bTls}, storage ${strStorage}, enc ${bEncrypt}, cmp ${strCompressType}")) {next} - - # Create hosts, file object, and config - my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup( - true, - {bHostBackup => $bRemote, bTls => $bTls, strStorage => $strStorage, bRepoEncrypt => $bEncrypt, - strCompressType => NONE}); - - # Reduce console logging to detail - $oHostDbPrimary->configUpdate({&CFGDEF_SECTION_GLOBAL => {'log-level-console' => lc(DETAIL)}}); - - # Create the wal path - my $strWalPath = $oHostDbPrimary->dbBasePath() . '/pg_xlog'; - storageTest()->pathCreate($strWalPath, {bCreateParent => true}); - - # Generate pg_control for stanza-create - storageTest()->pathCreate($oHostDbPrimary->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true}); - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_94); - - # Create archive-push command - my $strCommandPush = - $oHostDbPrimary->backrestExe() . ' --config=' . $oHostDbPrimary->backrestConfig() . ' --stanza=' . $self->stanza() . - ' archive-push'; - - my $strCommandGet = - $oHostDbPrimary->backrestExe() . ' --config=' . $oHostDbPrimary->backrestConfig() . ' --stanza=' . $self->stanza() . - ' archive-get'; - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' archive.info missing'); - my $strSourceFile1 = $self->walSegment(1, 1, 1); - storageTest()->pathCreate("${strWalPath}/archive_status"); - my $strArchiveFile1 = $self->walGenerate($strWalPath, PG_VERSION_94, 1, $strSourceFile1); - - $oHostDbPrimary->executeSimple( - $strCommandPush . " ${strWalPath}/${strSourceFile1}", {iExpectedExitStatus => ERROR_REPO_INVALID}); - - $oHostDbPrimary->executeSimple( - $strCommandGet . " ${strSourceFile1} ${strWalPath}/RECOVERYXLOG", {iExpectedExitStatus => ERROR_REPO_INVALID}); - - #--------------------------------------------------------------------------------------------------------------------------- - $oHostBackup->stanzaCreate('stanza create', {strOptionalParam => '--no-online'}); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' push first WAL'); - - my @stryExpectedWAL; - my $strSourceFile = $self->walSegment(1, 1, 1); - my $strArchiveFile = $self->walGenerate($strWalPath, PG_VERSION_94, 2, $strSourceFile); - - $oHostDbPrimary->executeSimple( - $strCommandPush . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . - " --compress-type=${strCompressType} ${strWalPath}/${strSourceFile}"); - push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.${strCompressType}"); - - # Test that the WAL was pushed - $self->archiveCheck($oHostBackup, $strSourceFile, $strArchiveChecksum, $strCompressType); - - # Remove from archive_status - storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready"); - - # Remove WAL - storageTest()->remove("${strWalPath}/${strSourceFile}", {bIgnoreMissing => false}); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' get missing WAL'); - - $oHostDbPrimary->executeSimple( - $strCommandGet . " 700000007000000070000000 ${strWalPath}/RECOVERYXLOG", {iExpectedExitStatus => 1}); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' get first WAL'); - - $oHostDbPrimary->executeSimple( - $strCommandGet . " ${strSourceFile} ${strWalPath}/RECOVERYXLOG"); - - # Check that the destination file exists - if (storageTest()->exists("${strWalPath}/RECOVERYXLOG")) - { - my ($strActualChecksum) = storageTest()->hashSize("${strWalPath}/RECOVERYXLOG"); - - if ($strActualChecksum ne $strArchiveChecksum) - { - confess "recovered file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'"; - } - } - else - { - confess "archive file '${strWalPath}/RECOVERYXLOG' is not in destination"; - } - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' push second WAL'); - - # Generate second WAL segment - $strSourceFile = $self->walSegment(1, 1, 2); - $strArchiveFile = $self->walGenerate($strWalPath, PG_VERSION_94, 2, $strSourceFile); - - # Create a temp file to make sure it is deleted later (skip when S3 since it doesn't use temp files) - my $strArchiveTmp; - - if ($strStorage eq POSIX) - { - # Should succeed when temp file already exists - &log(INFO, ' succeed when tmp WAL file exists'); - - $strArchiveTmp = - $oHostBackup->repoPath() . '/archive/' . $self->stanza() . '/' . PG_VERSION_94 . '-1/' . - substr($strSourceFile, 0, 16) . "/${strSourceFile}-${strArchiveChecksum}.${strCompressType}" . qw{.} . - STORAGE_TEMP_EXT; - - storageTest()->put($strArchiveTmp, 'JUNK'); - } - - # Push the WAL - $oHostDbPrimary->executeSimple( - "${strCommandPush} --compress-type=${strCompressType} --archive-async --process-max=2 ${strWalPath}/${strSourceFile}"); - push(@stryExpectedWAL, "${strSourceFile}-${strArchiveChecksum}.${strCompressType}"); - - # Make sure the temp file no longer exists if it was created - if (defined($strArchiveTmp)) - { - my $oWait = waitInit(5); - my $bFound = true; - - do - { - $bFound = storageTest()->exists($strArchiveTmp); - } - while ($bFound && waitMore($oWait)); - - if ($bFound) - { - confess "${strArchiveTmp} should have been removed by archive command"; - } - } - - # Test that the WAL was pushed - $self->archiveCheck($oHostBackup, $strSourceFile, $strArchiveChecksum, $strCompressType, $oHostDbPrimary->spoolPath()); - - # Remove from archive_status - storageTest()->remove("${strWalPath}/archive_status/${strSourceFile}.ready"); - - # Remove from spool - storageTest()->remove($oHostDbPrimary->spoolPath() . '/archive/' . $self->stanza() . "/out/${strSourceFile}.ok"); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' push history file'); - - storageTest()->put("${strWalPath}/00000002.history", 'HISTORYDATA'); - storageTest()->put("${strWalPath}/archive_status/00000002.history.ready", undef); - - $oHostDbPrimary->executeSimple("${strCommandPush} --archive-async ${strWalPath}/00000002.history"); - - if (!storageRepo()->exists($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/00000002.history'))) - { - confess 'unable to find history file in archive'; - } - - storageTest()->remove("${strWalPath}/archive_status/00000002.history.ready"); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' db version mismatch error - archive-get unable to retrieve archiveId'); - - # db section and corresponding history munged - $oHostBackup->infoMunge( - $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE), - {&INFO_ARCHIVE_SECTION_DB_HISTORY => {'1' => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}}); - - $oHostDbPrimary->executeSimple( - $strCommandGet . " ${strSourceFile1} ${strWalPath}/RECOVERYXLOG", {iExpectedExitStatus => ERROR_REPO_INVALID}); - - # Restore the file to its original condition - $oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE)); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' db system-id mismatch error'); - - $oHostBackup->infoMunge( - $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE), - {&INFO_ARCHIVE_SECTION_DB => {&INFO_BACKUP_KEY_SYSTEM_ID => 5000900090001855000}, - &INFO_ARCHIVE_SECTION_DB_HISTORY => {'1' => {&INFO_ARCHIVE_KEY_DB_ID => 5000900090001855000}}}); - - $oHostDbPrimary->executeSimple( - $strCommandPush . " ${strWalPath}/${strSourceFile}", {iExpectedExitStatus => ERROR_REPO_INVALID}); - - $oHostDbPrimary->executeSimple( - $strCommandGet . " ${strSourceFile1} ${strWalPath}/RECOVERYXLOG", {iExpectedExitStatus => ERROR_REPO_INVALID}); - - # Restore the file to its original condition - $oHostBackup->infoRestore($oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE)); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' stop'); - - $oHostDbPrimary->stop({strStanza => $oHostDbPrimary->stanza()}); - - $oHostDbPrimary->executeSimple($strCommandPush . " ${strWalPath}/${strSourceFile}", {iExpectedExitStatus => ERROR_STOP}); - - $oHostDbPrimary->executeSimple($strCommandGet . " ${strSourceFile1} ${strWalPath}/RECOVERYXLOG"); - - $oHostDbPrimary->start({strStanza => $oHostDbPrimary->stanza()}); - - storageTest->remove("${strWalPath}/RECOVERYXLOG", {bIgnoreMissing => false}); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' WAL duplicate ok'); - - $oHostDbPrimary->executeSimple($strCommandPush . " ${strWalPath}/${strSourceFile}"); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' WAL duplicate error'); - - $strArchiveFile = $self->walGenerate($strWalPath, PG_VERSION_94, 1, $strSourceFile); - - $oHostDbPrimary->executeSimple( - $strCommandPush . " ${strWalPath}/${strSourceFile}", {iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE}); - - # Remove WAL - storageTest()->remove("${strWalPath}/${strSourceFile}", {bIgnoreMissing => false}); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, " get second WAL (${strSourceFile})"); - - $oHostDbPrimary->executeSimple( - $strCommandGet . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . " --archive-async" . - ($strStorage eq POSIX ? " --repo-type=cifs" : '') . " ${strSourceFile} ${strWalPath}/RECOVERYXLOG"); - - # Check that the destination file exists - if (storageTest()->exists("${strWalPath}/RECOVERYXLOG")) - { - my ($strActualChecksum) = storageTest()->hashSize("${strWalPath}/RECOVERYXLOG"); - - if ($strActualChecksum ne $strArchiveChecksum) - { - confess "recovered file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'"; - } - } - else - { - confess "archive file '${strWalPath}/RECOVERYXLOG' is not in destination"; - } - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, " get history file"); - - $oHostDbPrimary->executeSimple( - $strCommandGet . " --archive-async 00000001.history ${strWalPath}/00000001.history", {iExpectedExitStatus => 1}); - - $oHostDbPrimary->executeSimple($strCommandGet . " --archive-async 00000002.history ${strWalPath}/00000002.history"); - - if (${storageTest()->get("${strWalPath}/00000002.history")} ne 'HISTORYDATA') - { - confess 'history contents do not match original'; - } - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' .partial WAL'); - - $strArchiveFile = $self->walGenerate($strWalPath, PG_VERSION_94, 2, "${strSourceFile}.partial"); - $oHostDbPrimary->executeSimple($strCommandPush . " ${strWalPath}/${strSourceFile}.partial"); - $self->archiveCheck($oHostBackup, "${strSourceFile}.partial", $strArchiveChecksum); - - push(@stryExpectedWAL, "${strSourceFile}.partial-${strArchiveChecksum}"); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' .partial WAL duplicate'); - - $oHostDbPrimary->executeSimple($strCommandPush . " ${strWalPath}/${strSourceFile}.partial"); - $self->archiveCheck($oHostBackup, "${strSourceFile}.partial", $strArchiveChecksum); - - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' .partial WAL with different checksum'); - - $strArchiveFile = $self->walGenerate($strWalPath, PG_VERSION_94, 1, "${strSourceFile}.partial"); - $oHostDbPrimary->executeSimple( - $strCommandPush . " ${strWalPath}/${strSourceFile}.partial", {iExpectedExitStatus => ERROR_ARCHIVE_DUPLICATE}); - - #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult( - sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_94 . '-1/0000000100000001'))}, - '(' . join(', ', @stryExpectedWAL) . ')', - 'all WAL in archive', {iWaitSeconds => 5}); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm deleted file mode 100644 index 6ec91d213..000000000 --- a/test/lib/pgBackRestTest/Module/Mock/MockExpireTest.pm +++ /dev/null @@ -1,205 +0,0 @@ -#################################################################################################################################### -# Tests for expire command -#################################################################################################################################### -package pgBackRestTest::Module::Mock::MockExpireTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::ExpireEnvTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostS3Test; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::Manifest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - use constant SECONDS_PER_DAY => 86400; - my $lBaseTime = time() - (SECONDS_PER_DAY * 56); - my $strDescription; - - foreach my $rhRun - ( - {vm => VM2, storage => S3, encrypt => true}, - {vm => VM3, storage => POSIX, encrypt => false}, - {vm => VM4, storage => AZURE, encrypt => true}, - ) - { - # Only run tests for this vm - next if ($rhRun->{vm} ne vmTest($self->vm())); - - # Increment the run, log, and decide whether this unit test should be run - my $strStorage = $rhRun->{storage}; - my $bEncrypt = $rhRun->{encrypt}; - - ############################################################################################################################ - if ($self->begin("simple, enc ${bEncrypt}, storage ${strStorage}")) - { - # Create hosts, file object, and config - my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup( - true, {strStorage => $strStorage, bRepoEncrypt => $bEncrypt}); - - # Create the test object - my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest( - $oHostBackup, $self->backrestExe(), storageRepo(), $oHostDbPrimary->dbPath(), $self); - - $oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_96); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Nothing to expire'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY, 246); - - $oExpireTest->process($self->stanza(), 1, 1, CFGOPTVAL_BACKUP_TYPE_FULL, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire oldest full backup, archive expire falls on segment major boundary'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), 1, 1, CFGOPTVAL_BACKUP_TYPE_FULL, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire oldest full backup'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY, 256); - $oExpireTest->process($self->stanza(), 1, 1, CFGOPTVAL_BACKUP_TYPE_FULL, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire oldest diff backup, archive expire does not fall on major segment boundary'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY, undef, 0); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY, undef, 0); - $oExpireTest->process($self->stanza(), 1, 1, CFGOPTVAL_BACKUP_TYPE_DIFF, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire oldest diff backup (cascade to incr)'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), 1, 1, CFGOPTVAL_BACKUP_TYPE_DIFF, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire archive based on newest incr backup'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), 1, 1, CFGOPTVAL_BACKUP_TYPE_INCR, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire diff treating full as diff'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), 2, 1, CFGOPTVAL_BACKUP_TYPE_DIFF, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire diff with repo-retention-archive with warning repo-retention-diff not set'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), undef, undef, CFGOPTVAL_BACKUP_TYPE_DIFF, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire full with repo-retention-archive with warning repo-retention-full not set'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), undef, undef, CFGOPTVAL_BACKUP_TYPE_FULL, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire no archive with warning since repo-retention-archive not set for INCR'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), 1, 1, CFGOPTVAL_BACKUP_TYPE_INCR, undef, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire no archive with warning since neither repo-retention-archive nor repo-retention-diff is set'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), undef, undef, CFGOPTVAL_BACKUP_TYPE_DIFF, undef, $strDescription); - } - - ############################################################################################################################ - if ($self->begin("stanzaUpgrade, enc ${bEncrypt}, storage ${strStorage}")) - { - # Create hosts, file object, and config - my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup( - true, {strStorage => $strStorage, bRepoEncrypt => $bEncrypt}); - - # Create the test object - my $oExpireTest = new pgBackRestTest::Env::ExpireEnvTest( - $oHostBackup, $self->backrestExe(), storageRepo(), $oHostDbPrimary->dbPath(), $self); - - $oExpireTest->stanzaCreate($self->stanza(), PG_VERSION_96); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Create backups in current db version'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), undef, undef, CFGOPTVAL_BACKUP_TYPE_DIFF, undef, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Upgrade stanza and expire only earliest db backup and archive'; - - $oExpireTest->stanzaUpgrade($self->stanza(), PG_VERSION_94); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_INCR, $lBaseTime += SECONDS_PER_DAY, 246); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_DIFF, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), 3, undef, CFGOPTVAL_BACKUP_TYPE_FULL, undef, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Upgrade the stanza, create full back - earliest db orphaned archive removed and earliest full backup ' . - 'and archive in previous db version removed'; - - $oExpireTest->stanzaUpgrade($self->stanza(), PG_VERSION_10); - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), 2, undef, CFGOPTVAL_BACKUP_TYPE_FULL, undef, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire all archive last full backup through pitr'; - - $oExpireTest->backupCreate($self->stanza(), CFGOPTVAL_BACKUP_TYPE_FULL, $lBaseTime += SECONDS_PER_DAY); - $oExpireTest->process($self->stanza(), 3, 1, CFGOPTVAL_BACKUP_TYPE_DIFF, 1, $strDescription); - - #----------------------------------------------------------------------------------------------------------------------- - $strDescription = 'Expire all archive except for the current database'; - - $oExpireTest->process($self->stanza(), 2, undef, CFGOPTVAL_BACKUP_TYPE_FULL, undef, $strDescription); - } - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm deleted file mode 100644 index 829b05e88..000000000 --- a/test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm +++ /dev/null @@ -1,249 +0,0 @@ -#################################################################################################################################### -# Mock Stanza Module Tests -#################################################################################################################################### -package pgBackRestTest::Module::Mock::MockStanzaTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Manifest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - foreach my $rhRun - ( - {vm => VM2, remote => false, tls => false, storage => AZURE, encrypt => true, compress => BZ2}, - {vm => VM2, remote => true, tls => true, storage => POSIX, encrypt => false, compress => GZ}, - {vm => VM3, remote => false, tls => false, storage => GCS, encrypt => false, compress => ZST}, - {vm => VM3, remote => true, tls => true, storage => AZURE, encrypt => true, compress => LZ4}, - {vm => VM4, remote => false, tls => false, storage => S3, encrypt => false, compress => GZ}, - {vm => VM4, remote => true, tls => false, storage => GCS, encrypt => true, compress => ZST}, - ) - { - # Only run tests for this vm - next if ($rhRun->{vm} ne vmTest($self->vm())); - - # Increment the run, log, and decide whether this unit test should be run - my $bRemote = $rhRun->{remote}; - my $bTls = $rhRun->{tls}; - my $strStorage = $rhRun->{storage}; - my $bEncrypt = $rhRun->{encrypt}; - my $strCompressType = $rhRun->{compress}; - - # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin("remote ${bRemote}, tls ${bTls}, storage ${strStorage}, enc ${bEncrypt}, cmp ${strCompressType}")) {next} - - # Create hosts, file object, and config - my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup( - true, - {bHostBackup => $bRemote, bTls => $bTls, strStorage => $strStorage, bRepoEncrypt => $bEncrypt, - strCompressType => $strCompressType}); - - # Archive and backup info file names - my $strArchiveInfoFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE); - my $strArchiveInfoCopyFile = $oHostBackup->repoArchivePath(ARCHIVE_INFO_FILE . INI_COPY_EXT); - my $strArchiveInfoOldFile = "${strArchiveInfoFile}.old"; - my $strArchiveInfoCopyOldFile = "${strArchiveInfoCopyFile}.old"; - - my $strBackupInfoFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO); - my $strBackupInfoCopyFile = $oHostBackup->repoBackupPath(FILE_BACKUP_INFO . INI_COPY_EXT); - my $strBackupInfoOldFile = "${strBackupInfoFile}.old"; - my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old"; - - # Create the stanza - $oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_MISSING, - strOptionalParam => '--no-online --log-level-file=info'}); - - # Generate pg_control for stanza-create - storageTest()->pathCreate(($oHostDbPrimary->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true}); - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_94); - - # Fail stanza upgrade before stanza-create has been performed - #-------------------------------------------------------------------------------------------------------------------------- - $oHostBackup->stanzaUpgrade('fail on stanza not initialized since archive.info is missing', - {iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-online'}); - - # Create the stanza successfully without force - run from the db server to ensure ability to run remotely - #-------------------------------------------------------------------------------------------------------------------------- - $oHostDbPrimary->stanzaCreate('successfully create the stanza', {strOptionalParam => '--no-online'}); - - # Rerun stanza-create and confirm it does not fail - #-------------------------------------------------------------------------------------------------------------------------- - $oHostDbPrimary->stanzaCreate( - 'do not fail on rerun of stanza-create - info files exist and DB section ok', - {strOptionalParam => '--no-online'}); - - # Stanza Create fails when not using force - database mismatch with pg_control file - #-------------------------------------------------------------------------------------------------------------------------- - # Change the database version by copying a new pg_control file - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_95); - - $oHostBackup->stanzaCreate('fail on database mismatch and warn force option deprecated', - {iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-online --force'}); - - # Restore pg_control - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_94); - - # Perform a stanza upgrade which will indicate already up to date - #-------------------------------------------------------------------------------------------------------------------------- - $oHostBackup->stanzaUpgrade('already up to date', {strOptionalParam => '--no-online'}); - - # Create the wal path - my $strWalPath = $oHostDbPrimary->dbBasePath() . '/pg_xlog'; - storageTest()->pathCreate("${strWalPath}/archive_status", {bCreateParent => true}); - - # Stanza Create fails - missing archive.info from non-empty archive dir - #-------------------------------------------------------------------------------------------------------------------------- - # Generate WAL then push to get valid archive data in the archive directory - my $strArchiveFile = $self->walSegment(1, 1, 1); - my $strSourceFile = $self->walGenerate($strWalPath, PG_VERSION_94, 1, $strArchiveFile); - - my $strCommand = $oHostDbPrimary->backrestExe() . ' --config=' . $oHostDbPrimary->backrestConfig() . - ' --stanza=db archive-push'; - $oHostDbPrimary->executeSimple($strCommand . " ${strSourceFile}"); - - # With data existing in the archive dir, move the info files and confirm failure - forceStorageMove(storageRepo(), $strArchiveInfoFile, $strArchiveInfoOldFile, {bRecurse => false}); - forceStorageMove(storageRepo(), $strArchiveInfoCopyFile, $strArchiveInfoCopyOldFile, {bRecurse => false}); - - if (!$bEncrypt) - { - $oHostBackup->stanzaCreate('fail on archive info file missing from non-empty dir', - {iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-online'}); - } - - # Restore info files from copy - forceStorageMove(storageRepo(), $strArchiveInfoOldFile, $strArchiveInfoFile, {bRecurse => false}); - forceStorageMove(storageRepo(), $strArchiveInfoCopyOldFile, $strArchiveInfoCopyFile, {bRecurse => false}); - - # Just before upgrading push one last WAL on the old version to ensure it can be retrieved later - #-------------------------------------------------------------------------------------------------------------------------- - $strArchiveFile = $self->walSegment(1, 1, 2); - $strSourceFile = $self->walGenerate($strWalPath, PG_VERSION_94, 1, $strArchiveFile); - $oHostDbPrimary->executeSimple($strCommand . " ${strSourceFile}"); - - # Fail on archive push due to mismatch of DB since stanza not upgraded - #-------------------------------------------------------------------------------------------------------------------------- - my $strArchiveTestFile = $self->testPath() . '/test-wal'; - storageTest()->put($strArchiveTestFile, $self->walGenerateContent(PG_VERSION_95)); - - # Upgrade the DB by copying new pg_control - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_95); - forceStorageMode(storageTest(), $oHostDbPrimary->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600'); - - # Fail on attempt to push an archive - $oHostDbPrimary->archivePush($strWalPath, $strArchiveTestFile, 1, ERROR_REPO_INVALID); - - # Perform a successful stanza upgrade noting additional history lines in info files for new version of the database - #-------------------------------------------------------------------------------------------------------------------------- - # Save a pre-upgrade copy of archive info for testing db-id mismatch - forceStorageMove(storageRepo(), $strArchiveInfoCopyFile, $strArchiveInfoCopyOldFile, {bRecurse => false}); - - $oHostBackup->stanzaUpgrade('successful upgrade creates additional history', {strOptionalParam => '--no-online'}); - - # Make sure that WAL from the old version can still be retrieved - #-------------------------------------------------------------------------------------------------------------------------- - # Generate the old pg_control so it looks like the original db has been restored - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_94); - - # Attempt to get the last archive log that was pushed to this repo - $oHostDbPrimary->executeSimple( - $oHostDbPrimary->backrestExe() . ' --config=' . $oHostDbPrimary->backrestConfig() . - " --stanza=db archive-get ${strArchiveFile} " . $oHostDbPrimary->dbBasePath() . '/pg_xlog/RECOVERYXLOG'); - - # Copy the new pg_control back so the tests can continue with the upgraded stanza - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_95); - forceStorageMode(storageTest(), $oHostDbPrimary->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600'); - - # After stanza upgrade, make sure archives are pushed to the new db verion-id directory (9.4-2) - #-------------------------------------------------------------------------------------------------------------------------- - # Push a WAL segment so have a valid file in the latest DB archive dir only - $oHostDbPrimary->archivePush($strWalPath, $strArchiveTestFile, 1); - $self->testResult( - sub {storageRepo()->list($oHostBackup->repoArchivePath(PG_VERSION_95 . '-2/0000000100000001'))}, - '000000010000000100000001-' . $self->walGenerateContentChecksum(PG_VERSION_95) . ".${strCompressType}", - 'check that WAL is in the archive at -2'); - - # Create the tablespace directory and perform a backup - #-------------------------------------------------------------------------------------------------------------------------- - storageTest()->pathCreate($oHostDbPrimary->dbBasePath() . '/' . DB_PATH_PGTBLSPC); - $oHostBackup->backup( - 'full', 'create first full backup ', {strOptionalParam => '--repo1-retention-full=2 --no-online'}, false); - - # Upgrade the stanza - #-------------------------------------------------------------------------------------------------------------------------- - # Copy pg_control for 9.6 - $self->controlGenerate($oHostDbPrimary->dbBasePath(), PG_VERSION_96); - forceStorageMode(storageTest(), $oHostDbPrimary->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600'); - - # Run from the db server to ensure ability to run remotely - $oHostDbPrimary->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-online'}); - - # Copy archive.info and restore really old version - forceStorageMove(storageRepo(), $strArchiveInfoFile, $strArchiveInfoOldFile, {bRecurse => false}); - forceStorageRemove(storageRepo(), $strArchiveInfoCopyFile, {bRecurse => false}); - forceStorageMove(storageRepo(), $strArchiveInfoCopyOldFile, $strArchiveInfoFile, {bRecurse => false}); - - # Confirm versions - my $oArchiveInfo = new pgBackRestTest::Env::ArchiveInfo($oHostBackup->repoArchivePath()); - my $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - $self->testResult(sub {$oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, - PG_VERSION_94)}, true, 'archive at old pg version'); - $self->testResult(sub {$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, - PG_VERSION_96)}, true, 'backup at new pg version'); - - $oHostBackup->stanzaUpgrade( - 'upgrade fails with mismatched db-ids', {iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-online'}); - - # Restore archive.info - forceStorageMove(storageRepo(), $strArchiveInfoOldFile, $strArchiveInfoFile, {bRecurse => false}); - - # Push a WAL and create a backup in the new DB to confirm diff changed to full - #-------------------------------------------------------------------------------------------------------------------------- - storageTest()->put($strArchiveTestFile, $self->walGenerateContent(PG_VERSION_96)); - $oHostDbPrimary->archivePush($strWalPath, $strArchiveTestFile, 1); - - # Test backup is changed from type=DIFF to FULL (WARN message displayed) - my $oExecuteBackup = $oHostBackup->backupBegin( - 'diff', 'diff changed to full backup', {strOptionalParam => '--repo1-retention-full=2 --no-online'}); - $oHostBackup->backupEnd('full', $oExecuteBackup, undef, false); - - # Delete the stanza - #-------------------------------------------------------------------------------------------------------------------------- - $oHostDbPrimary->stanzaDelete('fail on missing stop file', {iExpectedExitStatus => ERROR_FILE_MISSING}); - - $oHostDbPrimary->stop({strStanza => $self->stanza()}); - $oHostDbPrimary->stanzaDelete('successfully delete the stanza'); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm b/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm deleted file mode 100644 index eead4f5ad..000000000 --- a/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm +++ /dev/null @@ -1,81 +0,0 @@ -#################################################################################################################################### -# Archive Performance Tests -#################################################################################################################################### -package pgBackRestTest::Module::Performance::PerformanceArchivePerlTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Storable qw(dclone); -use Time::HiRes qw(gettimeofday); - -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# initModule -#################################################################################################################################### -sub initModule -{ - my $self = shift; - - $self->{strSpoolPath} = $self->testPath() . '/spool'; - $self->{strLogPath} = $self->testPath() . '/log'; -} - -#################################################################################################################################### -# initTest -#################################################################################################################################### -sub initTest -{ - my $self = shift; - - # Create spool path - storageTest()->pathCreate($self->{strSpoolPath}, {bIgnoreExists => true, bCreateParent => true}); - storageTest()->pathCreate($self->{strLogPath}, {bIgnoreExists => true, bCreateParent => true}); -} - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - ################################################################################################################################ - if ($self->begin("archive-push async (detect ok file)")) - { - #--------------------------------------------------------------------------------------------------------------------------- - storageTest()->put( - storageTest()->openWrite( - 'spool/archive/' . $self->stanza() . '/out/000000010000000100000001.ok', {bPathCreate => true})); - - my $iRunTotal = 100; - my $lTimeBegin = gettimeofday(); - - for (my $iIndex = 0; $iIndex < $iRunTotal; $iIndex++) - { - my $iResult = system( - $self->backrestExe() . ' --stanza=' . $self->stanza() . ' --archive-async --spool-path=' . $self->{strSpoolPath} . - ' --log-level-file=detail --log-path=' . $self->{strLogPath} . ' --archive-timeout=1' . - ' --pg1-path=/not/used archive-push /pg_xlog/000000010000000100000001'); - - if ($iResult != 0) - { - confess "archive-push returned ${iResult}"; - } - } - - &log(INFO, 'time per execution: ' . ((gettimeofday() - $lTimeBegin) / $iRunTotal)); - } -} - -1;