From 89d3476e325a94cb75457354e26b1ab4f5bcc8b0 Mon Sep 17 00:00:00 2001 From: David Steele Date: Sun, 29 Apr 2018 10:16:59 -0400 Subject: [PATCH] Refactor archive common functions in preparation for parallel async archive-get. --- lib/pgBackRest/Archive/Base.pm | 102 --------- lib/pgBackRest/Archive/Common.pm | 65 ++++++ lib/pgBackRest/Archive/Get/File.pm | 202 ++++++++++++++++++ lib/pgBackRest/Archive/Get/Get.pm | 105 +-------- lib/pgBackRest/Archive/Push/Async.pm | 72 +------ lib/pgBackRest/Archive/Push/File.pm | 4 +- lib/pgBackRest/Archive/Push/Push.pm | 8 - lib/pgBackRest/Check/Check.pm | 4 +- lib/pgBackRest/Protocol/Helper.pm | 2 - lib/pgBackRest/Protocol/Remote/Minion.pm | 6 +- src/Makefile | 1 + src/command/archive/common.c | 96 +++++++++ src/command/archive/common.h | 14 ++ src/command/archive/push/push.c | 98 +-------- test/define.yaml | 14 +- test/expect/mock-archive-001.log | 8 +- test/expect/mock-archive-002.log | 8 +- test/expect/mock-stanza-001.log | 8 +- test/expect/mock-stanza-002.log | 8 +- test/expect/mock-stanza-003.log | 8 +- .../Module/Archive/ArchiveCommonPerlTest.pm | 40 ++++ .../Module/Archive/ArchiveGetPerlTest.pm | 33 ++- .../Module/Archive/ArchivePushPerlTest.pm | 42 +--- test/src/module/archive/commonTest.c | 90 ++++++++ test/src/module/archive/pushTest.c | 81 ------- 25 files changed, 575 insertions(+), 544 deletions(-) create mode 100644 lib/pgBackRest/Archive/Get/File.pm create mode 100644 src/command/archive/common.c create mode 100644 src/command/archive/common.h create mode 100644 test/src/module/archive/commonTest.c diff --git a/lib/pgBackRest/Archive/Base.pm b/lib/pgBackRest/Archive/Base.pm index 491746b17..0fee5739e 100644 --- a/lib/pgBackRest/Archive/Base.pm +++ b/lib/pgBackRest/Archive/Base.pm @@ -54,106 +54,4 @@ sub new ); } -#################################################################################################################################### -# getCheck - Given a specific database version and system-id, find a file in the archive. If no database info was passed, the -# current database will be used. -#################################################################################################################################### -sub getCheck -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $strFile, - $bCheck, - ) = - logDebugParam - ( - __PACKAGE__ . '->getCheck', \@_, - {name => 'strDbVersion', required => false}, - {name => 'ullDbSysId', required => false}, - {name => 'strFile', required => false}, - {name => 'bCheck', required => false, default => true}, - ); - - my @stryArchiveId = (); - my $strArchiveId; - my $strArchiveFile; - my $strCipherPass; - - # If the dbVersion/dbSysId are not passed, then we need to retrieve the database information - if (!defined($strDbVersion) || !defined($ullDbSysId) ) - { - # get DB info for comparison - ($strDbVersion, my $iControlVersion, my $iCatalogVersion, $ullDbSysId) = dbMasterGet()->info(); - } - - # Get db info from the repo - if (!isRepoLocal()) - { - ($strArchiveId, $strArchiveFile, $strCipherPass) = protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP)->cmdExecute( - OP_ARCHIVE_GET_CHECK, [$strDbVersion, $ullDbSysId, $strFile, $bCheck], true); - } - else - { - my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), true); - - # Check that the archive info is compatible with the database if required (not required for archive-get) - if ($bCheck) - { - push(@stryArchiveId, $oArchiveInfo->check($strDbVersion, $ullDbSysId)); - } - # Else if the database version and system-id are in the info history list then get a list of corresponding archiveIds - else - { - @stryArchiveId = $oArchiveInfo->archiveIdList($strDbVersion, $ullDbSysId); - } - - # Default the returned archiveId to the newest in the event the WAL segment is not found then the most recent archiveID will - # be returned. If none were found, then the preceding calls will error. - $strArchiveId = $stryArchiveId[0]; - - # If a file was passed to look for, then look for the file starting in the newest matching archiveId to the oldest - if (defined($strFile)) - { - foreach my $strId (@stryArchiveId) - { - # Then if it is a WAL segment, try to find it - if (walIsSegment($strFile)) - { - $strArchiveFile = walSegmentFind(storageRepo(), $strId, $strFile); - } - # Else if not a WAL segment, see if it exists in the archive dir - elsif (storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/${strId}/${strFile}")) - { - $strArchiveFile = $strFile; - } - - # If the file was found, then return the archiveId where it was found - if (defined($strArchiveFile)) - { - $strArchiveId = $strId; - last; - } - } - } - - # Get the encryption passphrase to read/write files (undefined if the repo is not encrypted) - $strCipherPass = $oArchiveInfo->cipherPassSub(); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveId', value => $strArchiveId}, - {name => 'strArchiveFile', value => $strArchiveFile}, - {name => 'strCipherPass', value => $strCipherPass, redact => true} - ); -} - 1; diff --git a/lib/pgBackRest/Archive/Common.pm b/lib/pgBackRest/Archive/Common.pm index 94193f908..23807a6f0 100644 --- a/lib/pgBackRest/Archive/Common.pm +++ b/lib/pgBackRest/Archive/Common.pm @@ -44,6 +44,14 @@ use constant PG_WAL_SYSTEM_ID_OFFSET_LT_93 => 12; use constant PG_WAL_SEGMENT_SIZE => 16777216; push @EXPORT, qw(PG_WAL_SEGMENT_SIZE); +#################################################################################################################################### +# WAL status constants +#################################################################################################################################### +use constant WAL_STATUS_ERROR => 'error'; + push @EXPORT, qw(WAL_STATUS_ERROR); +use constant WAL_STATUS_OK => 'ok'; + push @EXPORT, qw(WAL_STATUS_OK); + #################################################################################################################################### # PostgreSQL WAL magic #################################################################################################################################### @@ -425,4 +433,61 @@ sub walIsPartial push @EXPORT, qw(walIsPartial); +#################################################################################################################################### +# archiveAsyncStatusWrite +# +# Write out a status file to the spool path with information about the success or failure of an archive push. +#################################################################################################################################### +sub archiveAsyncStatusWrite +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strType, + $strSpoolPath, + $strWalFile, + $iCode, + $strMessage, + ) = + logDebugParam + ( + __PACKAGE__ . '::archiveAsyncStatusWrite', \@_, + {name => 'strType'}, + {name => 'strSpoolPath'}, + {name => 'strWalFile'}, + {name => 'iCode', required => false}, + {name => 'strMessage', required => false}, + ); + + # Remove any error file exists unless a new one will be written + if ($strType ne WAL_STATUS_ERROR) + { + # Remove the error file, if any + storageLocal()->remove("${strSpoolPath}/${strWalFile}.error", {bIgnoreMissing => true}); + } + + # Write the status file + my $strStatus; + + if (defined($iCode)) + { + if (!defined($strMessage)) + { + confess &log(ASSERT, 'strMessage must be set when iCode is set'); + } + + $strStatus = "${iCode}\n${strMessage}"; + } + elsif ($strType eq WAL_STATUS_ERROR) + { + confess &log(ASSERT, 'error status must have iCode and strMessage set'); + } + + storageLocal()->put( + storageLocal()->openWrite("${strSpoolPath}/${strWalFile}.${strType}", {bAtomic => true}), $strStatus); +} + +push @EXPORT, qw(archiveAsyncStatusWrite); + 1; diff --git a/lib/pgBackRest/Archive/Get/File.pm b/lib/pgBackRest/Archive/Get/File.pm new file mode 100644 index 000000000..c0c3b1f95 --- /dev/null +++ b/lib/pgBackRest/Archive/Get/File.pm @@ -0,0 +1,202 @@ +#################################################################################################################################### +# ARCHIVE GET FILE MODULE +#################################################################################################################################### +package pgBackRest::Archive::Get::File; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Exporter qw(import); + our @EXPORT = qw(); +use File::Basename qw(basename dirname); + +use pgBackRest::Archive::Common; +use pgBackRest::Archive::Info; +use pgBackRest::Db; +use pgBackRest::Common::Exception; +use pgBackRest::Common::Lock; +use pgBackRest::Common::Log; +use pgBackRest::Config::Config; +use pgBackRest::Protocol::Helper; +use pgBackRest::Protocol::Storage::Helper; +use pgBackRest::Storage::Base; +use pgBackRest::Storage::Filter::Gzip; +use pgBackRest::Storage::Filter::Sha; +use pgBackRest::Storage::Helper; + +#################################################################################################################################### +# Given a specific database version and system-id, find a file in the archive. If no database info was passed, the current database +# will be used. +#################################################################################################################################### +sub archiveGetCheck +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strDbVersion, + $ullDbSysId, + $strFile, + $bCheck, + ) = + logDebugParam + ( + __PACKAGE__ . '::archiveGetCheck', \@_, + {name => 'strDbVersion', required => false}, + {name => 'ullDbSysId', required => false}, + {name => 'strFile', required => false}, + {name => 'bCheck', required => false, default => true}, + ); + + my @stryArchiveId = (); + my $strArchiveId; + my $strArchiveFile; + my $strCipherPass; + + # If the dbVersion/dbSysId are not passed, then we need to retrieve the database information + if (!defined($strDbVersion) || !defined($ullDbSysId) ) + { + # get DB info for comparison + ($strDbVersion, my $iControlVersion, my $iCatalogVersion, $ullDbSysId) = dbMasterGet()->info(); + } + + # Get db info from the repo + if (!isRepoLocal()) + { + ($strArchiveId, $strArchiveFile, $strCipherPass) = protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP)->cmdExecute( + OP_ARCHIVE_GET_CHECK, [$strDbVersion, $ullDbSysId, $strFile, $bCheck], true); + } + else + { + my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), true); + + # Check that the archive info is compatible with the database if required (not required for archive-get) + if ($bCheck) + { + push(@stryArchiveId, $oArchiveInfo->check($strDbVersion, $ullDbSysId)); + } + # Else if the database version and system-id are in the info history list then get a list of corresponding archiveIds + else + { + @stryArchiveId = $oArchiveInfo->archiveIdList($strDbVersion, $ullDbSysId); + } + + # Default the returned archiveId to the newest in the event the WAL segment is not found then the most recent archiveID will + # be returned. If none were found, then the preceding calls will error. + $strArchiveId = $stryArchiveId[0]; + + # If a file was passed to look for, then look for the file starting in the newest matching archiveId to the oldest + if (defined($strFile)) + { + foreach my $strId (@stryArchiveId) + { + # Then if it is a WAL segment, try to find it + if (walIsSegment($strFile)) + { + $strArchiveFile = walSegmentFind(storageRepo(), $strId, $strFile); + } + # Else if not a WAL segment, see if it exists in the archive dir + elsif (storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/${strId}/${strFile}")) + { + $strArchiveFile = $strFile; + } + + # If the file was found, then return the archiveId where it was found + if (defined($strArchiveFile)) + { + $strArchiveId = $strId; + last; + } + } + } + + # Get the encryption passphrase to read/write files (undefined if the repo is not encrypted) + $strCipherPass = $oArchiveInfo->cipherPassSub(); + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strArchiveId', value => $strArchiveId}, + {name => 'strArchiveFile', value => $strArchiveFile}, + {name => 'strCipherPass', value => $strCipherPass, redact => true} + ); +} + +push @EXPORT, qw(archiveGetCheck); + +#################################################################################################################################### +# archiveGetFile +# +# Copy a file from the archive. +#################################################################################################################################### +sub archiveGetFile +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strSourceArchive, + $strDestinationFile + ) = + logDebugParam + ( + __PACKAGE__ . '::archiveGetFile', \@_, + {name => 'strSourceArchive'}, + {name => 'strDestinationFile'} + ); + + lockStopTest(); + + # Get the repo storage + my $oStorageRepo = storageRepo(); + + # Construct absolute path to the WAL file when it is relative + $strDestinationFile = walPath($strDestinationFile, cfgOption(CFGOPT_PG_PATH, false), cfgCommandName(cfgCommandGet())); + + # Get the wal segment filename + my ($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(undef, undef, $strSourceArchive, false); + + # If there are no matching archive files then there are two possibilities: + # 1) The end of the archive stream has been reached, this is normal and a 1 will be returned + # 2) There is a hole in the archive stream and a hard error should be returned. However, holes are possible due to async + # archiving - so when to report a hole? Since a hard error will cause PG to terminate, for now treat as case #1. + my $iResult = 0; + + if (!defined($strArchiveFile)) + { + &log(INFO, "unable to find ${strSourceArchive} in the archive"); + + $iResult = 1; + } + else + { + # Determine if the source file is already compressed + my $bSourceCompressed = $strArchiveFile =~ ('^.*\.' . COMPRESS_EXT . '$') ? true : false; + + # Copy the archive file to the requested location + # If the file is encrypted, then the passphrase from the info file is required to open the archive file in the repo + $oStorageRepo->copy( + $oStorageRepo->openRead( + STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strArchiveFile}", {bProtocolCompress => !$bSourceCompressed, + strCipherPass => defined($strCipherPass) ? $strCipherPass : undef}), + storageDb()->openWrite( + $strDestinationFile, + {rhyFilter => $bSourceCompressed ? + [{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef})); + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'iResult', value => $iResult} + ); +} + +push @EXPORT, qw(archiveGetFile); + +1; diff --git a/lib/pgBackRest/Archive/Get/Get.pm b/lib/pgBackRest/Archive/Get/Get.pm index 3af25462a..abec4e2e2 100644 --- a/lib/pgBackRest/Archive/Get/Get.pm +++ b/lib/pgBackRest/Archive/Get/Get.pm @@ -19,6 +19,7 @@ use pgBackRest::Common::Exception; use pgBackRest::Common::Lock; use pgBackRest::Common::Log; use pgBackRest::Archive::Common; +use pgBackRest::Archive::Get::File; use pgBackRest::Archive::Info; use pgBackRest::Common::String; use pgBackRest::Common::Wait; @@ -77,109 +78,7 @@ sub process return logDebugReturn ( $strOperation, - {name => 'iResult', value => $self->get($strSourceArchive, $strDestinationFile), trace => true} - ); -} - -#################################################################################################################################### -# get -#################################################################################################################################### -sub get -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strSourceArchive, - $strDestinationFile - ) = - logDebugParam - ( - __PACKAGE__ . '->get', \@_, - {name => 'strSourceArchive'}, - {name => 'strDestinationFile'} - ); - - lockStopTest(); - - # Get the repo storage - my $oStorageRepo = storageRepo(); - - # Construct absolute path to the WAL file when it is relative - $strDestinationFile = walPath($strDestinationFile, cfgOption(CFGOPT_PG_PATH, false), cfgCommandName(cfgCommandGet())); - - # Get the wal segment filename - my ($strArchiveId, $strArchiveFile, $strCipherPass) = $self->getCheck(undef, undef, $strSourceArchive, false); - - # If there are no matching archive files then there are two possibilities: - # 1) The end of the archive stream has been reached, this is normal and a 1 will be returned - # 2) There is a hole in the archive stream and a hard error should be returned. However, holes are possible due to async - # archiving - so when to report a hole? Since a hard error will cause PG to terminate, for now treat as case #1. - my $iResult = 0; - - if (!defined($strArchiveFile)) - { - &log(INFO, "unable to find ${strSourceArchive} in the archive"); - - $iResult = 1; - } - else - { - # Determine if the source file is already compressed - my $bSourceCompressed = $strArchiveFile =~ ('^.*\.' . COMPRESS_EXT . '$') ? true : false; - - # Copy the archive file to the requested location - # If the file is encrypted, then the passphrase from the info file is required to open the archive file in the repo - $oStorageRepo->copy( - $oStorageRepo->openRead( - STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strArchiveFile}", {bProtocolCompress => !$bSourceCompressed, - strCipherPass => defined($strCipherPass) ? $strCipherPass : undef}), - storageDb()->openWrite( - $strDestinationFile, - {rhyFilter => $bSourceCompressed ? - [{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef})); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iResult', value => $iResult} - ); -} - -#################################################################################################################################### -# getArchiveId -# -# CAUTION: Only to be used by commands where the DB Version and DB System ID are not important such that the pg-path is not valid -# for the command (i.e. expire command). Since this function will not check validity of the database version call getCheck() -# instead. -#################################################################################################################################### -sub getArchiveId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->getArchiveId'); - - my $strArchiveId; - - if (!isRepoLocal()) - { - $strArchiveId = protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP)->cmdExecute(OP_ARCHIVE_GET_ARCHIVE_ID, undef, true); - } - else - { - $strArchiveId = (new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), true))->archiveId(); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveId', value => $strArchiveId, trace => true} + {name => 'iResult', value => archiveGetFile($strSourceArchive, $strDestinationFile), trace => true}, ); } diff --git a/lib/pgBackRest/Archive/Push/Async.pm b/lib/pgBackRest/Archive/Push/Async.pm index 3a4712a9d..20d2ae7f6 100644 --- a/lib/pgBackRest/Archive/Push/Async.pm +++ b/lib/pgBackRest/Archive/Push/Async.pm @@ -169,8 +169,9 @@ sub processQueue # If error then write out an error file if (defined($hJob->{oException})) { - $self->walStatusWrite( - WAL_STATUS_ERROR, $strWalFile, $hJob->{oException}->code(), $hJob->{oException}->message()); + archiveAsyncStatusWrite( + WAL_STATUS_ERROR, $self->{strSpoolPath}, $strWalFile, $hJob->{oException}->code(), + $hJob->{oException}->message()); $iErrorTotal++; @@ -181,8 +182,8 @@ sub processQueue # Else write success else { - $self->walStatusWrite( - WAL_STATUS_OK, $strWalFile, defined($strWarning) ? 0 : undef, + archiveAsyncStatusWrite( + WAL_STATUS_OK, $self->{strSpoolPath}, $strWalFile, defined($strWarning) ? 0 : undef, defined($strWarning) ? $strWarning : undef); $iOkTotal++; @@ -200,8 +201,8 @@ sub processQueue { foreach my $strDropFile (@{$stryDropList}) { - $self->walStatusWrite( - WAL_STATUS_OK, $strDropFile, 0, + archiveAsyncStatusWrite( + WAL_STATUS_OK, $self->{strSpoolPath}, $strDropFile, 0, "dropped WAL file ${strDropFile} because archive queue exceeded " . cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) . ' bytes'); @@ -224,8 +225,8 @@ sub processQueue # Error all queued jobs foreach my $strWalFile (@{$stryWalFile}) { - $self->walStatusWrite( - WAL_STATUS_ERROR, $strWalFile, $iCode, $strMessage); + archiveAsyncStatusWrite( + WAL_STATUS_ERROR, $self->{strSpoolPath}, $strWalFile, $iCode, $strMessage); } } } @@ -240,59 +241,4 @@ sub processQueue ); } -#################################################################################################################################### -# walStatusWrite -# -# Write out a status file to the spool path with information about the success or failure of an archive push. -#################################################################################################################################### -sub walStatusWrite -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strWalFile, - $iCode, - $strMessage, - ) = - logDebugParam - ( - __PACKAGE__ . '->writeStatus', \@_, - {name => 'strType'}, - {name => 'strWalFile'}, - {name => 'iCode', required => false}, - {name => 'strMessage', required => false}, - ); - - # Remove any error file exists unless a new one will be written - if ($strType ne WAL_STATUS_ERROR) - { - # Remove the error file, if any - storageSpool()->remove("$self->{strSpoolPath}/${strWalFile}.error", {bIgnoreMissing => true}); - } - - # Write the status file - my $strStatus; - - if (defined($iCode)) - { - if (!defined($strMessage)) - { - confess &log(ASSERT, 'strMessage must be set when iCode is set'); - } - - $strStatus = "${iCode}\n${strMessage}"; - } - elsif ($strType eq WAL_STATUS_ERROR) - { - confess &log(ASSERT, 'error status must have iCode and strMessage set'); - } - - storageSpool()->put( - storageSpool()->openWrite("$self->{strSpoolPath}/${strWalFile}.${strType}", {bAtomic => true}), $strStatus); -} - 1; diff --git a/lib/pgBackRest/Archive/Push/File.pm b/lib/pgBackRest/Archive/Push/File.pm index e39cb9e60..00f90cdb0 100644 --- a/lib/pgBackRest/Archive/Push/File.pm +++ b/lib/pgBackRest/Archive/Push/File.pm @@ -26,8 +26,8 @@ use pgBackRest::Storage::Helper; #################################################################################################################################### # archivePushCheck # -# Check that a WAL segment does not already exist in the archive be pushing. Files that are not segments (e.g. .history, .backup) -# will always be reported as not present and will be overwritten by archivePushFile(). +# Check that a WAL segment does not already exist in the archive before pushing. Files that are not segments (e.g. .history, +# .backup) will always be reported as not present and will be overwritten by archivePushFile(). #################################################################################################################################### sub archivePushCheck { diff --git a/lib/pgBackRest/Archive/Push/Push.pm b/lib/pgBackRest/Archive/Push/Push.pm index c82f3acba..93f6440a9 100644 --- a/lib/pgBackRest/Archive/Push/Push.pm +++ b/lib/pgBackRest/Archive/Push/Push.pm @@ -24,14 +24,6 @@ use pgBackRest::Protocol::Helper; use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Helper; -#################################################################################################################################### -# WAL status constants -#################################################################################################################################### -use constant WAL_STATUS_ERROR => 'error'; - push @EXPORT, qw(WAL_STATUS_ERROR); -use constant WAL_STATUS_OK => 'ok'; - push @EXPORT, qw(WAL_STATUS_OK); - #################################################################################################################################### # process # diff --git a/lib/pgBackRest/Check/Check.pm b/lib/pgBackRest/Check/Check.pm index abf58df59..bfe57f455 100644 --- a/lib/pgBackRest/Check/Check.pm +++ b/lib/pgBackRest/Check/Check.pm @@ -9,7 +9,7 @@ use Carp qw(confess); use English '-no_match_vars'; use pgBackRest::Archive::Common; -use pgBackRest::Archive::Get::Get; +use pgBackRest::Archive::Get::File; use pgBackRest::Backup::Info; use pgBackRest::Common::Exception; use pgBackRest::Common::Log; @@ -150,7 +150,7 @@ sub process eval { # Check that the archive info file is written and is valid for the current database of the stanza - ($strArchiveId) = new pgBackRest::Archive::Get::Get()->getCheck(); + ($strArchiveId) = archiveGetCheck(); return true; } or do diff --git a/lib/pgBackRest/Protocol/Helper.pm b/lib/pgBackRest/Protocol/Helper.pm index 16c6f24aa..fcee6b3af 100644 --- a/lib/pgBackRest/Protocol/Helper.pm +++ b/lib/pgBackRest/Protocol/Helper.pm @@ -23,8 +23,6 @@ use constant OP_BACKUP_FILE => 'backupF push @EXPORT, qw(OP_BACKUP_FILE); # Archive Module -use constant OP_ARCHIVE_GET_ARCHIVE_ID => 'archiveId'; - push @EXPORT, qw(OP_ARCHIVE_GET_ARCHIVE_ID); use constant OP_ARCHIVE_GET_CHECK => 'archiveCheck'; push @EXPORT, qw(OP_ARCHIVE_GET_CHECK); use constant OP_ARCHIVE_PUSH_CHECK => 'archivePushCheck'; diff --git a/lib/pgBackRest/Protocol/Remote/Minion.pm b/lib/pgBackRest/Protocol/Remote/Minion.pm index 8a7996feb..3a7d7a7b9 100644 --- a/lib/pgBackRest/Protocol/Remote/Minion.pm +++ b/lib/pgBackRest/Protocol/Remote/Minion.pm @@ -14,7 +14,7 @@ use pgBackRest::Backup::File; use pgBackRest::Common::Log; use pgBackRest::Common::Io::Buffered; use pgBackRest::Common::Wait; -use pgBackRest::Archive::Get::Get; +use pgBackRest::Archive::Get::File; use pgBackRest::Archive::Push::File; use pgBackRest::Check::Check; use pgBackRest::Config::Config; @@ -70,7 +70,6 @@ sub init # Create objects my $oStorage = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_DB) ? storageDb() : storageRepo(); - my $oArchiveGet = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_BACKUP) ? new pgBackRest::Archive::Get::Get() : undef; my $oCheck = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_BACKUP) ? new pgBackRest::Check::Check() : undef; my $oInfo = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_BACKUP) ? new pgBackRest::Info() : undef; my $oDb = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_DB) ? new pgBackRest::Db() : undef; @@ -79,8 +78,7 @@ sub init my $hCommandMap = { # ArchiveGet commands - &OP_ARCHIVE_GET_ARCHIVE_ID => sub {$oArchiveGet->getArchiveId()}, - &OP_ARCHIVE_GET_CHECK => sub {$oArchiveGet->getCheck(@{shift()})}, + &OP_ARCHIVE_GET_CHECK => sub {archiveGetCheck(@{shift()})}, # ArchivePush commands &OP_ARCHIVE_PUSH_CHECK => sub {archivePushCheck(@{shift()})}, diff --git a/src/Makefile b/src/Makefile index 147aa296e..3edc2c487 100644 --- a/src/Makefile +++ b/src/Makefile @@ -54,6 +54,7 @@ DESTDIR = # List of required source files. main.c should always be listed last and the rest in alpha order. #################################################################################################################################### SRCS = \ + command/archive/common.c \ command/archive/push/push.c \ command/help/help.c \ command/command.c \ diff --git a/src/command/archive/common.c b/src/command/archive/common.c new file mode 100644 index 000000000..42048d494 --- /dev/null +++ b/src/command/archive/common.c @@ -0,0 +1,96 @@ +/*********************************************************************************************************************************** +Archive Push Command +***********************************************************************************************************************************/ +#include + +#include "command/archive/common.h" +#include "common/log.h" +#include "common/memContext.h" +#include "common/wait.h" +#include "storage/helper.h" + +/*********************************************************************************************************************************** +Check for ok/error status files in the spool in/out directory +***********************************************************************************************************************************/ +bool +archiveAsyncStatus(const String *walSegment, bool confessOnError) +{ + bool result = false; + + MEM_CONTEXT_TEMP_BEGIN() + { + StringList *fileList = storageListP( + storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT), .expression = strNewFmt("^%s\\.(ok|error)$", strPtr(walSegment))); + + if (fileList != NULL && strLstSize(fileList) > 0) + { + // If more than one status file was found then assert - this could be a bug in the async process + if (strLstSize(fileList) != 1) + { + THROW( + AssertError, "multiple status files found in '%s' for WAL segment '%s'", + strPtr(storagePathNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT))), strPtr(walSegment)); + } + + // Get the status file content + const String *statusFile = strLstGet(fileList, 0); + + String *content = strNewBuf( + storageGetNP(storageNewReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile))))); + + // Get the code and message if the file has content + int code = 0; + const String *message = NULL; + + if (strSize(content) != 0) + { + // Find the line feed after the error code -- should be the first one + const char *linefeedPtr = strchr(strPtr(content), '\n'); + + // Error if linefeed not found + if (linefeedPtr == NULL) + THROW(FormatError, "%s content must have at least two lines", strPtr(statusFile)); + + // Error if message is zero-length + if (strlen(linefeedPtr + 1) == 0) + THROW(FormatError, "%s message must be > 0", strPtr(statusFile)); + + // Get contents + code = varIntForce(varNewStr(strNewN(strPtr(content), (size_t)(linefeedPtr - strPtr(content))))); + message = strTrim(strNew(linefeedPtr + 1)); + } + + // Process OK files + if (strEndsWithZ(statusFile, ".ok")) + { + // If there is content in the status file it is a warning + if (strSize(content) != 0) + { + // If error code is not success, then this was a renamed .error file + if (code != 0) + { + message = strNewFmt( + "WAL segment '%s' was not pushed due to error [%d] and was manually skipped: %s", strPtr(walSegment), + code, strPtr(message)); + } + + LOG_WARN(strPtr(message)); + } + + result = true; + } + else if (confessOnError) + { + // Error status files must have content + if (strSize(content) == 0) + THROW(AssertError, "status file '%s' has no content", strPtr(statusFile)); + + // Throw error using the code passed in the file + THROW_CODE(code, strPtr(message)); + } + } + } + MEM_CONTEXT_TEMP_END(); + + return result; +} diff --git a/src/command/archive/common.h b/src/command/archive/common.h new file mode 100644 index 000000000..45054a497 --- /dev/null +++ b/src/command/archive/common.h @@ -0,0 +1,14 @@ +/*********************************************************************************************************************************** +Archive Common +***********************************************************************************************************************************/ +#ifndef COMMAND_ARCHIVE_COMMON_H +#define COMMAND_ARCHIVE_COMMON_H + +#include "common/type/string.h" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +bool archiveAsyncStatus(const String *walSegment, bool confessOnError); + +#endif diff --git a/src/command/archive/push/push.c b/src/command/archive/push/push.c index a35fe288f..df97981ff 100644 --- a/src/command/archive/push/push.c +++ b/src/command/archive/push/push.c @@ -1,113 +1,19 @@ /*********************************************************************************************************************************** Archive Push Command ***********************************************************************************************************************************/ -#include -#include -#include -#include -#include -#include -#include #include +#include "command/archive/common.h" #include "command/command.h" #include "common/fork.h" -#include "common/error.h" #include "common/log.h" #include "common/memContext.h" -#include "common/regExp.h" #include "common/wait.h" #include "config/config.h" #include "config/load.h" #include "perl/exec.h" #include "storage/helper.h" -/*********************************************************************************************************************************** -Check for ok/error status files in the spool out directory -***********************************************************************************************************************************/ -static bool -walStatus(const String *walSegment, bool confessOnError) -{ - bool result = false; - - MEM_CONTEXT_TEMP_BEGIN() - { - StringList *fileList = storageListP( - storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT), .expression = strNewFmt("^%s\\.(ok|error)$", strPtr(walSegment))); - - if (fileList != NULL && strLstSize(fileList) > 0) - { - // If more than one status file was found then assert - this could be a bug in the async process - if (strLstSize(fileList) != 1) - { - THROW( - AssertError, "multiple status files found in '%s' for WAL segment '%s'", - strPtr(storagePathNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT))), strPtr(walSegment)); - } - - // Get the status file content - const String *statusFile = strLstGet(fileList, 0); - - String *content = strNewBuf( - storageGetNP(storageNewReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile))))); - - // Get the code and message if the file has content - int code = 0; - const String *message = NULL; - - if (strSize(content) != 0) - { - // Find the line feed after the error code -- should be the first one - const char *linefeedPtr = strchr(strPtr(content), '\n'); - - // Error if linefeed not found - if (linefeedPtr == NULL) - THROW(FormatError, "%s content must have at least two lines", strPtr(statusFile)); - - // Error if message is zero-length - if (strlen(linefeedPtr + 1) == 0) - THROW(FormatError, "%s message must be > 0", strPtr(statusFile)); - - // Get contents - code = varIntForce(varNewStr(strNewN(strPtr(content), (size_t)(linefeedPtr - strPtr(content))))); - message = strTrim(strNew(linefeedPtr + 1)); - } - - // Process OK files - if (strEndsWithZ(statusFile, ".ok")) - { - // If there is content in the status file it is a warning - if (strSize(content) != 0) - { - // If error code is not success, then this was a renamed .error file - if (code != 0) - { - message = strNewFmt( - "WAL segment '%s' was not pushed due to error [%d] and was manually skipped: %s", strPtr(walSegment), - code, strPtr(message)); - } - - LOG_WARN(strPtr(message)); - } - - result = true; - } - else if (confessOnError) - { - // Error status files must have content - if (strSize(content) == 0) - THROW(AssertError, "status file '%s' has no content", strPtr(statusFile)); - - // Throw error using the code passed in the file - THROW_CODE(code, strPtr(message)); - } - } - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - /*********************************************************************************************************************************** Push a WAL segment to the repository ***********************************************************************************************************************************/ @@ -139,7 +45,7 @@ cmdArchivePush() { // Check if the WAL segment has been pushed. Errors will not be confessed on the first try to allow the async // process a chance to fix them. - pushed = walStatus(walSegment, confessOnError); + pushed = archiveAsyncStatus(walSegment, confessOnError); // If the WAL segment has not already been pushed then start the async process to push it. There's no point in // forking the async process off more than once so track that as well. Use an archive lock to prevent more than diff --git a/test/define.yaml b/test/define.yaml index 5c3190c31..d2cad184e 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -497,16 +497,23 @@ unit: - name: archive test: + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common + total: 1 + + coverage: + command/archive/common: full + # ---------------------------------------------------------------------------------------------------------------------------- - name: common-perl - total: 4 + total: 5 coverage: Archive/Common: partial # ---------------------------------------------------------------------------------------------------------------------------- - name: push-perl - total: 8 + total: 7 coverage: Archive/Push/Async: full @@ -516,7 +523,7 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: push - total: 2 + total: 1 perlReq: true coverage: @@ -535,6 +542,7 @@ unit: coverage: Archive/Base: partial + Archive/Get/File: partial Archive/Get/Get: partial # ******************************************************************************************************************************** diff --git a/test/expect/mock-archive-001.log b/test/expect/mock-archive-001.log index 641a761f4..8104aa81d 100644 --- a/test/expect/mock-archive-001.log +++ b/test/expect/mock-archive-001.log @@ -116,12 +116,12 @@ P00 INFO: archive-push command end: completed successfully P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001 P00 INFO: get WAL segment 000000010000000100000001 -P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001 +P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001 P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = , strCipherType = aes-256-cbc, strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp -P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000001, ullDbSysId = [undef] +P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000001, ullDbSysId = [undef] P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true P00 DEBUG: Db->new(): iRemoteIdx = 1 P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] @@ -143,13 +143,13 @@ P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStor P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000100000001, strSortOrder = P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113) P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113 -P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113, strArchiveId = 9.4-1, strCipherPass = +P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113, strArchiveId = 9.4-1, strCipherPass = P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], strCipherPass = , xFileExp = /9.4-1/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113 P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] -P00 DEBUG: Archive::Get::Get->get=>: iResult = 0 +P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0 P00 DEBUG: Main::mainCleanup(): iExitCode = 0 P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 diff --git a/test/expect/mock-archive-002.log b/test/expect/mock-archive-002.log index 2e9fe0753..0d841fd63 100644 --- a/test/expect/mock-archive-002.log +++ b/test/expect/mock-archive-002.log @@ -100,7 +100,7 @@ P00 INFO: archive-push command end: completed successfully P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001 P00 INFO: get WAL segment 000000010000000100000001 -P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001 +P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001 P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup @@ -110,7 +110,7 @@ P00 DEBUG: Protocol::Helper::protocolParam=>: strRemoteCommand = [BACKREST- P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 3, iCompressLevelNetwork = 1, iProtocolTimeout = 60, iSshPort = [undef], strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1] P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 3, iCompressLevelNetwork = 1, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no pgbackrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --stanza=db --type=backup remote', strId = remote process on 'backup', strName = remote P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] -P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000001, ullDbSysId = [undef] +P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000001, ullDbSysId = [undef] P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true P00 DEBUG: Db->new(): iRemoteIdx = 1 P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] @@ -118,13 +118,13 @@ P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 1000000000000000094 P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol -P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113, strArchiveId = 9.4-1, strCipherPass = [undef] +P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113, strArchiveId = 9.4-1, strCipherPass = [undef] P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /9.4-1/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113 P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] -P00 DEBUG: Archive::Get::Get->get=>: iResult = 0 +P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0 P00 DEBUG: Main::mainCleanup(): iExitCode = 0 P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup diff --git a/test/expect/mock-stanza-001.log b/test/expect/mock-stanza-001.log index 6be85d82d..5464b8631 100644 --- a/test/expect/mock-stanza-001.log +++ b/test/expect/mock-stanza-001.log @@ -518,12 +518,12 @@ db-version="9.4" P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 P00 INFO: get WAL segment 000000010000000100000002 -P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 +P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp -P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef] +P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef] P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true P00 DEBUG: Db->new(): iRemoteIdx = 1 P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] @@ -545,13 +545,13 @@ P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStor P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000002-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.3-1/0000000100000001, strSortOrder = P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz) P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz -P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = [undef] +P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = [undef] P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], strCipherPass = [undef], xFileExp = /9.3-1/000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] -P00 DEBUG: Archive::Get::Get->get=>: iResult = 0 +P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0 P00 DEBUG: Main::mainCleanup(): iExitCode = 0 P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 diff --git a/test/expect/mock-stanza-002.log b/test/expect/mock-stanza-002.log index 96853ecf8..c717b5836 100644 --- a/test/expect/mock-stanza-002.log +++ b/test/expect/mock-stanza-002.log @@ -384,7 +384,7 @@ db-version="9.4" P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-2] --stanza=db P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 P00 INFO: get WAL segment 000000010000000100000002 -P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 +P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup @@ -394,7 +394,7 @@ P00 DEBUG: Protocol::Helper::protocolParam=>: strRemoteCommand = [BACKREST- P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 3, iCompressLevelNetwork = 1, iProtocolTimeout = 60, iSshPort = [undef], strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-2] P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 3, iCompressLevelNetwork = 1, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no pgbackrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --stanza=db --type=backup remote', strId = remote process on 'backup', strName = remote P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] -P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef] +P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef] P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true P00 DEBUG: Db->new(): iRemoteIdx = 1 P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] @@ -402,13 +402,13 @@ P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> P00 DEBUG: Db->info=>: iDbCatalogVersion = 201306121, iDbControlVersion = 937, strDbVersion = 9.3, ullDbSysId = 1000000000000000093 P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol -P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = +P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /9.3-1/000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] -P00 DEBUG: Archive::Get::Get->get=>: iResult = 0 +P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0 P00 DEBUG: Main::mainCleanup(): iExitCode = 0 P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup diff --git a/test/expect/mock-stanza-003.log b/test/expect/mock-stanza-003.log index 093fa8c48..b0d5b1a69 100644 --- a/test/expect/mock-stanza-003.log +++ b/test/expect/mock-stanza-003.log @@ -493,12 +493,12 @@ db-version="9.4" P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key= --repo1-s3-key-secret= --repo1-s3-region=us-east-1 --no-repo1-s3-verify-ssl --repo1-type=s3 --stanza=db P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 P00 INFO: get WAL segment 000000010000000100000002 -P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 +P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false P00 DEBUG: Storage::S3::Request->new(): bVerifySsl = false, iPort = [undef], lBufferMax = 4194304, strAccessKeyId = , strBucket = pgbackrest-dev, strCaFile = [undef], strCaPath = [undef], strEndPoint = s3.amazonaws.com, strHost = [undef], strRegion = us-east-1, strSecretAccessKey = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp -P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef] +P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef] P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true P00 DEBUG: Db->new(): iRemoteIdx = 1 P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] @@ -520,13 +520,13 @@ P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStor P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000002-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.3-1/0000000100000001, strSortOrder = P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz) P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz -P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = [undef] +P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = [undef] P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], strCipherPass = [undef], xFileExp = /9.3-1/000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] -P00 DEBUG: Archive::Get::Get->get=>: iResult = 0 +P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0 P00 DEBUG: Main::mainCleanup(): iExitCode = 0 P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 diff --git a/test/lib/pgBackRestTest/Module/Archive/ArchiveCommonPerlTest.pm b/test/lib/pgBackRestTest/Module/Archive/ArchiveCommonPerlTest.pm index 5c41a9d1f..18d51a012 100644 --- a/test/lib/pgBackRestTest/Module/Archive/ArchiveCommonPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Archive/ArchiveCommonPerlTest.pm @@ -156,6 +156,46 @@ sub run $self->testResult( sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found"); } + + ################################################################################################################################ + if ($self->begin("archiveAsyncStatusWrite()")) + { + my $iWalTimeline = 1; + my $iWalMajor = 1; + my $iWalMinor = 1; + + # Create the spool path + my $strSpoolPath = $self->testPath() . "/spool/out"; + $self->storageTest()->pathCreate($strSpoolPath, {bIgnoreExists => true, bCreateParent => true}); + + #--------------------------------------------------------------------------------------------------------------------------- + my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++); + + # Generate a normal ok + archiveAsyncStatusWrite(WAL_STATUS_OK, $strSpoolPath, $strSegment); + + #--------------------------------------------------------------------------------------------------------------------------- + # Generate a valid warning ok + archiveAsyncStatusWrite(WAL_STATUS_OK, $strSpoolPath, $strSegment, 0, 'Test Warning'); + + #--------------------------------------------------------------------------------------------------------------------------- + # Generate an invalid error + $self->testException( + sub {archiveAsyncStatusWrite(WAL_STATUS_ERROR, $strSpoolPath, $strSegment)}, ERROR_ASSERT, + "error status must have iCode and strMessage set"); + + #--------------------------------------------------------------------------------------------------------------------------- + # Generate an invalid error + $self->testException( + sub {archiveAsyncStatusWrite(WAL_STATUS_ERROR, $strSpoolPath, $strSegment, ERROR_ASSERT)}, + ERROR_ASSERT, "strMessage must be set when iCode is set"); + + #--------------------------------------------------------------------------------------------------------------------------- + # Generate a valid error + archiveAsyncStatusWrite( + WAL_STATUS_ERROR, $strSpoolPath, $strSegment, ERROR_ARCHIVE_DUPLICATE, + "WAL segment ${strSegment} already exists in the archive"); + } } 1; diff --git a/test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm b/test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm index 2870e8e57..8190e38ee 100644 --- a/test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm @@ -15,6 +15,7 @@ use Storable qw(dclone); use Digest::SHA qw(sha1_hex); use pgBackRest::Archive::Common; +use pgBackRest::Archive::Get::File; use pgBackRest::Archive::Get::Get; use pgBackRest::Archive::Info; use pgBackRest::Common::Exception; @@ -91,8 +92,6 @@ sub run ################################################################################################################################ if ($self->begin("Archive::Base::getCheck()")) { - my $oArchiveBase = new pgBackRest::Archive::Base(); - # Create and save archive.info file my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false, {bLoad => false, bIgnoreMissing => true}); @@ -104,39 +103,39 @@ sub run # db-version, db-sys-id passed but combination doesn't exist in archive.info history #--------------------------------------------------------------------------------------------------------------------------- - $self->testException(sub {$oArchiveBase->getCheck( + $self->testException(sub {archiveGetCheck( PG_VERSION_95, $self->dbSysId(PG_VERSION_94), undef, false)}, ERROR_UNKNOWN, "unable to retrieve the archive id for database version '" . PG_VERSION_95 . "' and system-id '" . $self->dbSysId(PG_VERSION_94) . "'"); # db-version, db-sys-id and wal passed all undefined #--------------------------------------------------------------------------------------------------------------------------- - my ($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck(undef, undef, undef, false); + my ($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(undef, undef, undef, false); $self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)}, true, 'undef db-version, db-sys-id and wal returns only current db archive-id'); # db-version defined, db-sys-id and wal undefined #--------------------------------------------------------------------------------------------------------------------------- - ($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck(PG_VERSION_92, undef, undef, false); + ($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(PG_VERSION_92, undef, undef, false); $self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)}, true, 'old db-version, db-sys-id and wal undefined returns only current db archive-id'); # db-version undefined, db-sys-id defined and wal undefined #--------------------------------------------------------------------------------------------------------------------------- - ($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck( + ($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck( undef, $self->dbSysId(PG_VERSION_93), undef, false); $self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)}, true, 'undef db-version, old db-sys-id and wal undef returns only current db archive-id'); # old db-version, db-sys-id and wal undefined, check = true (default) #--------------------------------------------------------------------------------------------------------------------------- - ($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck(PG_VERSION_92, undef, undef); + ($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(PG_VERSION_92, undef, undef); $self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)}, true, 'old db-version, db-sys-id and wal undefined, check = true returns only current db archive-id'); # old db-version, old db-sys-id and wal undefined, check = true (default) #--------------------------------------------------------------------------------------------------------------------------- - $self->testException(sub {$oArchiveBase->getCheck( + $self->testException(sub {archiveGetCheck( PG_VERSION_93, $self->dbSysId(PG_VERSION_93), undef)}, ERROR_ARCHIVE_MISMATCH, "WAL segment version " . PG_VERSION_93 . " does not match archive version " . PG_VERSION_94 . "\n" . "WAL segment system-id " . $self->dbSysId(PG_VERSION_93) . " does not match archive system-id " . @@ -151,7 +150,7 @@ sub run storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true}); storageRepo()->put("${strWalMajorPath}/${strWalSegmentName}"); - ($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck(undef, undef, $strWalSegment, false); + ($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(undef, undef, $strWalSegment, false); $self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)}, true, 'undef db-version, db-sys-id with a requested wal not in current db archive returns only current db archive-id'); @@ -159,7 +158,7 @@ sub run # Pass db-version and db-sys-id where WAL is actually located #--------------------------------------------------------------------------------------------------------------------------- ($strArchiveId, $strArchiveFile, $strCipherPass) = - $oArchiveBase->getCheck(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), $strWalSegment, false); + archiveGetCheck(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), $strWalSegment, false); $self->testResult( sub {($strArchiveId eq PG_VERSION_92 . '-1') && ($strArchiveFile eq $strWalSegmentName) && !defined($strCipherPass)}, @@ -176,7 +175,7 @@ sub run storageRepo()->put("${strWalMajorPath}/${strWalSegmentName}", $strFileContent); ($strArchiveId, $strArchiveFile, $strCipherPass) = - $oArchiveBase->getCheck(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), $strWalSegment, false); + archiveGetCheck(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), $strWalSegment, false); # Using the returned values, confirm the correct file is read $self->testResult(sub {sha1_hex(${storageRepo()->get($self->{strArchivePath} . "/" . $strArchiveId . "/" . @@ -189,7 +188,7 @@ sub run { # archive.info missing #--------------------------------------------------------------------------------------------------------------------------- - $self->testException(sub {new pgBackRest::Archive::Get::Get()->get($strWalSegment, $strDestinationFile)}, + $self->testException(sub {archiveGetFile($strWalSegment, $strDestinationFile)}, ERROR_FILE_MISSING, ARCHIVE_INFO_FILE . " does not exist but is required to push/get WAL segments\n" . "HINT: is archive_command configured in postgresql.conf?\n" . @@ -206,7 +205,7 @@ sub run # file not found #--------------------------------------------------------------------------------------------------------------------------- - $self->testResult(sub {new pgBackRest::Archive::Get::Get()->get($strWalSegment, $strDestinationFile)}, 1, + $self->testResult(sub {archiveGetFile($strWalSegment, $strDestinationFile)}, 1, "unable to find ${strWalSegment} in the archive"); # file found but is not a WAL segment @@ -220,7 +219,7 @@ sub run # Create path to copy file storageRepo()->pathCreate($strDestinationPath); - $self->testResult(sub {new pgBackRest::Archive::Get::Get()->get(BOGUS, $strDestinationFile)}, 0, + $self->testResult(sub {archiveGetFile(BOGUS, $strDestinationFile)}, 0, "non-WAL segment copied"); # Confirm the correct file is copied @@ -245,14 +244,14 @@ sub run storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true}); storageRepo()->put("${strWalMajorPath}/${strWalSegmentName}", $strFileContent); - $self->testResult(sub {new pgBackRest::Archive::Get::Get()->get($strWalSegmentName, $strDestinationFile)}, 0, + $self->testResult(sub {archiveGetFile($strWalSegmentName, $strDestinationFile)}, 0, "WAL segment copied"); # Confirm the correct file is copied $self->testResult(sub {sha1_hex(${storageRepo()->get($strDestinationFile)})}, $strFileHash, ' check correct WAL copied when in multiple locations'); - # get files from an older DB version to simulate restoring from an old backup set to a database that is of that same version + # Get files from an older DB version to simulate restoring from an old backup set to a database that is of that same version #--------------------------------------------------------------------------------------------------------------------------- # Create same WAL name in older DB archive but with different data to ensure it is copied $strArchivePath = $self->{strArchivePath} . "/" . PG_VERSION_93 . "-2/"; @@ -272,7 +271,7 @@ sub run # Overwrite current pg_control file with older version $self->controlGenerate($self->{strDbPath}, PG_VERSION_93); - $self->testResult(sub {new pgBackRest::Archive::Get::Get()->get($strWalSegmentName, $strDestinationFile)}, 0, + $self->testResult(sub {archiveGetFile($strWalSegmentName, $strDestinationFile)}, 0, "WAL segment copied from older db backupset to same version older db"); # Confirm the correct file is copied diff --git a/test/lib/pgBackRestTest/Module/Archive/ArchivePushPerlTest.pm b/test/lib/pgBackRestTest/Module/Archive/ArchivePushPerlTest.pm index d42d269a4..6c5640b88 100644 --- a/test/lib/pgBackRestTest/Module/Archive/ArchivePushPerlTest.pm +++ b/test/lib/pgBackRestTest/Module/Archive/ArchivePushPerlTest.pm @@ -323,47 +323,6 @@ sub run $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); } - ################################################################################################################################ - if ($self->begin("ArchivePushAsync->walStatusWrite()")) - { - my $oPush = new pgBackRest::Archive::Push::Push(); - - my $oPushAsync = new pgBackRest::Archive::Push::Async($self->{strWalPath}, $self->{strSpoolPath}); - $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); - $oPushAsync->initServer(); - - my $iWalTimeline = 1; - my $iWalMajor = 1; - my $iWalMinor = 1; - - #--------------------------------------------------------------------------------------------------------------------------- - my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++); - - # Generate a normal ok - $oPushAsync->walStatusWrite(WAL_STATUS_OK, $strSegment); - - #--------------------------------------------------------------------------------------------------------------------------- - # Generate a valid warning ok - $oPushAsync->walStatusWrite(WAL_STATUS_OK, $strSegment, 0, 'Test Warning'); - - #--------------------------------------------------------------------------------------------------------------------------- - # Generate an invalid error - $self->testException( - sub {$oPushAsync->walStatusWrite(WAL_STATUS_ERROR, $strSegment)}, ERROR_ASSERT, - "error status must have iCode and strMessage set"); - - #--------------------------------------------------------------------------------------------------------------------------- - # Generate an invalid error - $self->testException( - sub {$oPushAsync->walStatusWrite(WAL_STATUS_ERROR, $strSegment, ERROR_ASSERT)}, ERROR_ASSERT, - "strMessage must be set when iCode is set"); - - #--------------------------------------------------------------------------------------------------------------------------- - # Generate a valid error - $oPushAsync->walStatusWrite( - WAL_STATUS_ERROR, $strSegment, ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strSegment} already exists in the archive"); - } - ################################################################################################################################ if ($self->begin("ArchivePushAsync->process()")) { @@ -668,6 +627,7 @@ sub run $self->optionTestClear(CFGOPT_SPOOL_PATH); $self->configTestLoad(CFGCMD_ARCHIVE_PUSH); } + ################################################################################################################################ if ($self->begin("ArchivePushFile::archivePushFile - encryption")) { diff --git a/test/src/module/archive/commonTest.c b/test/src/module/archive/commonTest.c new file mode 100644 index 000000000..62fa4c6d3 --- /dev/null +++ b/test/src/module/archive/commonTest.c @@ -0,0 +1,90 @@ +/*********************************************************************************************************************************** +Test Archive Common +***********************************************************************************************************************************/ +#include + +#include "common/harnessConfig.h" + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void +testRun() +{ + // ***************************************************************************************************************************** + if (testBegin("archiveAsyncStatus()")) + { + StringList *argList = strLstNew(); + strLstAddZ(argList, "pgbackrest"); + strLstAdd(argList, strNewFmt("--spool-path=%s", testPath())); + strLstAddZ(argList, "--archive-async"); + strLstAddZ(argList, "--archive-timeout=1"); + strLstAddZ(argList, "--stanza=db"); + strLstAddZ(argList, "archive-push"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + // ------------------------------------------------------------------------------------------------------------------------- + String *segment = strNew("000000010000000100000001"); + + TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), false, "directory and status file not present"); + + // ------------------------------------------------------------------------------------------------------------------------- + mkdir(strPtr(strNewFmt("%s/archive", testPath())), 0750); + mkdir(strPtr(strNewFmt("%s/archive/db", testPath())), 0750); + mkdir(strPtr(strNewFmt("%s/archive/db/out", testPath())), 0750); + + TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), false, "status file not present"); + + // ------------------------------------------------------------------------------------------------------------------------- + storagePutNP( + storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), + bufNewStr(strNew(BOGUS_STR))); + TEST_ERROR( + archiveAsyncStatus(segment, false), FormatError, "000000010000000100000001.ok content must have at least two lines"); + + storagePutNP( + storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), + bufNewStr(strNew(BOGUS_STR "\n"))); + TEST_ERROR(archiveAsyncStatus(segment, false), FormatError, "000000010000000100000001.ok message must be > 0"); + + storagePutNP( + storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), + bufNewStr(strNew(BOGUS_STR "\nmessage"))); + TEST_ERROR(archiveAsyncStatus(segment, false), FormatError, "unable to convert str 'BOGUS' to int"); + + storagePutNP( + storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), + bufNewStr(strNew("0\nwarning"))); + TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), true, "ok file with warning"); + testLogResult("P00 WARN: warning"); + + storagePutNP( + storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), + bufNewStr(strNew("25\nerror"))); + TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), true, "error status renamed to ok"); + testLogResult( + "P00 WARN: WAL segment '000000010000000100000001' was not pushed due to error [25] and was manually skipped: error"); + + // ------------------------------------------------------------------------------------------------------------------------- + storagePutNP( + storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))), + bufNewStr(strNew(""))); + TEST_ERROR( + archiveAsyncStatus(segment, false), AssertError, + strPtr( + strNewFmt( + "multiple status files found in '%s/archive/db/out' for WAL segment '000000010000000100000001'", testPath()))); + + unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))))); + TEST_ERROR(archiveAsyncStatus(segment, true), AssertError, "status file '000000010000000100000001.error' has no content"); + + storagePutNP( + storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))), + bufNewStr(strNew("25\nmessage"))); + TEST_ERROR(archiveAsyncStatus(segment, true), AssertError, "message"); + + TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), false, "suppress error"); + + unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))))); + } +} diff --git a/test/src/module/archive/pushTest.c b/test/src/module/archive/pushTest.c index c075f7f24..bf85a9fe2 100644 --- a/test/src/module/archive/pushTest.c +++ b/test/src/module/archive/pushTest.c @@ -1,11 +1,6 @@ /*********************************************************************************************************************************** Test Archive Push Command ***********************************************************************************************************************************/ -#include - -#include "config/load.h" -#include "version.h" - #include "common/harnessConfig.h" /*********************************************************************************************************************************** @@ -14,82 +9,6 @@ Test Run void testRun() { - // ***************************************************************************************************************************** - if (testBegin("walStatus()")) - { - StringList *argList = strLstNew(); - strLstAddZ(argList, "pgbackrest"); - strLstAdd(argList, strNewFmt("--spool-path=%s", testPath())); - strLstAddZ(argList, "--archive-async"); - strLstAddZ(argList, "--archive-timeout=1"); - strLstAddZ(argList, "--stanza=db"); - strLstAddZ(argList, "archive-push"); - harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); - - // ------------------------------------------------------------------------------------------------------------------------- - String *segment = strNew("000000010000000100000001"); - - TEST_RESULT_BOOL(walStatus(segment, false), false, "directory and status file not present"); - - // ------------------------------------------------------------------------------------------------------------------------- - mkdir(strPtr(strNewFmt("%s/archive", testPath())), 0750); - mkdir(strPtr(strNewFmt("%s/archive/db", testPath())), 0750); - mkdir(strPtr(strNewFmt("%s/archive/db/out", testPath())), 0750); - - TEST_RESULT_BOOL(walStatus(segment, false), false, "status file not present"); - - // ------------------------------------------------------------------------------------------------------------------------- - storagePutNP( - storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), - bufNewStr(strNew(BOGUS_STR))); - TEST_ERROR(walStatus(segment, false), FormatError, "000000010000000100000001.ok content must have at least two lines"); - - storagePutNP( - storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), - bufNewStr(strNew(BOGUS_STR "\n"))); - TEST_ERROR(walStatus(segment, false), FormatError, "000000010000000100000001.ok message must be > 0"); - - storagePutNP( - storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), - bufNewStr(strNew(BOGUS_STR "\nmessage"))); - TEST_ERROR(walStatus(segment, false), FormatError, "unable to convert str 'BOGUS' to int"); - - storagePutNP( - storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), - bufNewStr(strNew("0\nwarning"))); - TEST_RESULT_BOOL(walStatus(segment, false), true, "ok file with warning"); - testLogResult("P00 WARN: warning"); - - storagePutNP( - storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))), - bufNewStr(strNew("25\nerror"))); - TEST_RESULT_BOOL(walStatus(segment, false), true, "error status renamed to ok"); - testLogResult( - "P00 WARN: WAL segment '000000010000000100000001' was not pushed due to error [25] and was manually skipped: error"); - - // ------------------------------------------------------------------------------------------------------------------------- - storagePutNP( - storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))), - bufNewStr(strNew(""))); - TEST_ERROR( - walStatus(segment, false), AssertError, - strPtr( - strNewFmt( - "multiple status files found in '%s/archive/db/out' for WAL segment '000000010000000100000001'", testPath()))); - - unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))))); - TEST_ERROR(walStatus(segment, true), AssertError, "status file '000000010000000100000001.error' has no content"); - - storagePutNP( - storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))), - bufNewStr(strNew("25\nmessage"))); - TEST_ERROR(walStatus(segment, true), AssertError, "message"); - - TEST_RESULT_BOOL(walStatus(segment, false), false, "suppress error"); - - unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))))); - } - // ***************************************************************************************************************************** if (testBegin("cmdArchivePush()")) {