diff --git a/.travis.yml b/.travis.yml index 71a974d3b..88ccb23ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,14 @@ env: before_install: - sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl python-pip + - | + # Install & Configure AWS CLI + pip install --upgrade --user awscli + aws configure set region us-east-1 + aws configure set aws_access_key_id accessKey1 + aws configure set aws_secret_access_key verySecretKey1 + aws help --version + aws configure list - | # Build Devel::Cover git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl diff --git a/README.md b/README.md index b8fa1f991..7266edfcf 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,10 @@ Tablespaces are fully supported and on restore tablespaces can be remapped to an File and directory links are supported for any file or directory in the PostgreSQL cluster. When restoring it is possible to restore all links to their original locations, remap some or all links, or restore some or all links as normal files or directories within the cluster directory. +### Amazon S3 Support + +pgBackRest repositories can be stored on Amazon S3 to allow for virtually unlimited capacity and retention. + ### Compatibility with PostgreSQL >= 8.3 pgBackRest includes support for versions down to 8.3, since older versions of PostgreSQL are still regularly utilized. diff --git a/bin/pgbackrest b/bin/pgbackrest index 2690fd684..26ca5316c 100755 --- a/bin/pgbackrest +++ b/bin/pgbackrest @@ -63,7 +63,7 @@ local $EVAL_ERROR = undef; eval optionSet(OPTION_LOG_LEVEL_STDERR, PROTOCOL, true); logLevelSet(OFF, OFF, optionGet(OPTION_LOG_LEVEL_STDERR)); - if (optionTest(OPTION_TYPE, BACKUP) && !-e optionGet(OPTION_REPO_PATH)) + if (optionTest(OPTION_TYPE, BACKUP) && !optionTest(OPTION_REPO_TYPE, REPO_TYPE_S3) && !-e optionGet(OPTION_REPO_PATH)) { confess &log(ERROR, 'repo-path \'' . optionGet(OPTION_REPO_PATH) . '\' does not exist', ERROR_PATH_MISSING); } @@ -169,7 +169,7 @@ local $EVAL_ERROR = undef; eval require pgBackRest::Protocol::Storage::Helper; pgBackRest::Protocol::Storage::Helper->import(); - if (isRepoLocal() && !storageRepo()->pathExists('')) + if (isRepoLocal() && !optionTest(OPTION_REPO_TYPE, REPO_TYPE_S3) && !storageRepo()->pathExists('')) { confess &log(ERROR, 'repo-path \'' . optionGet(OPTION_REPO_PATH) . '\' does not exist', ERROR_PATH_MISSING); } diff --git a/doc/lib/BackRestDoc/Common/DocExecute.pm b/doc/lib/BackRestDoc/Common/DocExecute.pm index 51b3f76b3..4aaaa87ab 100644 --- a/doc/lib/BackRestDoc/Common/DocExecute.pm +++ b/doc/lib/BackRestDoc/Common/DocExecute.pm @@ -572,6 +572,7 @@ sub backrestConfig my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}}); delete($$oConfigClean{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_LEVEL_STDERR}); delete($$oConfigClean{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_TIMESTAMP}); + delete($$oConfigClean{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_VERIFY_SSL}); if (keys(%{$$oConfigClean{&CONFIG_SECTION_GLOBAL}}) == 0) { diff --git a/doc/xml/index.xml b/doc/xml/index.xml index 2e6023292..3c6c8aef7 100644 --- a/doc/xml/index.xml +++ b/doc/xml/index.xml @@ -124,6 +124,12 @@

File and directory links are supported for any file or directory in the cluster. When restoring it is possible to restore all links to their original locations, remap some or all links, or restore some or all links as normal files or directories within the cluster directory.

+
+ Amazon S3 Support + +

repositories can be stored on Amazon S3 to allow for virtually unlimited capacity and retention.

+
+
Compatibility with <postgres/> >= 8.3 diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index fb667b51c..9a930596b 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -229,6 +229,71 @@ /backup/db/backrest + + + S3 repository access key. + + AWS key used to access this bucket. + + AKIAIOSFODNN7EXAMPLE + + + + + S3 repository secret access key. + + AWS secret key used to access this bucket. + + wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + + + + + S3 repository bucket. + + S3 bucket used to store the repository. + + repositories can be stored in the bucket root by setting repo-path=/ but it is usually best to specify a prefix, such as /repo, so logs and other AWS generated content can also be stored in the bucket. + + db-backup + + + + + S3 repository endpoint. + + The AWS end point should be valid for the selected region. + + s3.amazonaws.com + + + + + S3 repository host. + + Connect to a host other than the end point. This is typically used for testing. + + 127.0.0.1 + + + + + S3 repository region. + + The AWS region where the bucket was created. + + us-east-1 + + + + + Verify S3 server certificate. + + Disables verification of the S3 server certificate. This should only be used for testing or other scenarios where a certificate has been self-signed. + + n + + Type of storage used for the repository. @@ -237,6 +302,7 @@ cifs diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 9861dd8e1..9716c9248 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -186,6 +186,16 @@ + + + + + + +

Amazon S3 repository support.

+ + + diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index c469e8423..fa9b66551 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -170,6 +170,10 @@
Installation + + + + mkdir /root/pgbackrest-release-{[version]} @@ -1321,6 +1325,55 @@
+ +
+ S3 Support + +

supports storing repositories in Amazon S3.

+ + + Configure <proper>S3</proper> + + s3 + / + accessKey1 + verySecretKey1 + demo-bucket + s3.amazonaws.com + us-east-1 + n + + 4 + + +

Commands are run exactly as if the repository were stored on a local disk.

+ + + Create the stanza + + + aws s3 --no-verify-ssl mb s3://demo-bucket 2>&1 + + + + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-create + completed successfully + + + +

File creation time in S3 is relatively slow so commands benefit by increasing process-max to parallelize file creation.

+ + + Backup the {[postgres-cluster-demo]} cluster + + + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} + --log-level-console=info backup + no prior backup exists|full backup size + + +
+
Dedicated Backup Host diff --git a/lib/pgBackRest/Common/Http/Client.pm b/lib/pgBackRest/Common/Http/Client.pm new file mode 100644 index 000000000..d5baf3d3e --- /dev/null +++ b/lib/pgBackRest/Common/Http/Client.pm @@ -0,0 +1,283 @@ +#################################################################################################################################### +# HTTP Client +#################################################################################################################################### +package pgBackRest::Common::Http::Client; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Exporter qw(import); + our @EXPORT = qw(); +use IO::Socket::SSL; + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Io::Buffered; +use pgBackRest::Common::Log; +use pgBackRest::Common::String; +use pgBackRest::Common::Xml; +use pgBackRest::Common::Http::Common; + +#################################################################################################################################### +# Constants +#################################################################################################################################### +use constant HTTP_VERB_GET => 'GET'; + push @EXPORT, qw(HTTP_VERB_GET); +use constant HTTP_VERB_POST => 'POST'; + push @EXPORT, qw(HTTP_VERB_POST); +use constant HTTP_VERB_PUT => 'PUT'; + push @EXPORT, qw(HTTP_VERB_PUT); + +use constant HTTP_HEADER_CONTENT_LENGTH => 'content-length'; + push @EXPORT, qw(HTTP_HEADER_CONTENT_LENGTH); +use constant HTTP_HEADER_TRANSFER_ENCODING => 'transfer-encoding'; + push @EXPORT, qw(HTTP_HEADER_TRANSFER_ENCODING); + +#################################################################################################################################### +# new +#################################################################################################################################### +our @ISA = (); ## no critic (ClassHierarchies::ProhibitExplicitISA) + +sub new +{ + my $class = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strHost, + $strVerb, + $strUri, + $hQuery, + $hRequestHeader, + $rstrRequestBody, + $iProtocolTimeout, + $lBufferMax, + $bVerifySsl, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'strHost', trace => true}, + {name => 'strVerb', trace => true}, + {name => 'strUri', optional => true, default => qw(/), trace => true}, + {name => 'hQuery', optional => true, trace => true}, + {name => 'hRequestHeader', optional => true, trace => true}, + {name => 'rstrRequestBody', optional => true, trace => true}, + {name => 'iProtocolTimeout', optional => true, default => 30, trace => true}, + {name => 'lBufferMax', optional => true, default => 32768, trace => true}, + {name => 'bVerifySsl', optional => true, default => true, trace => true}, + ); + + # Connect to the server + my $oSocket = IO::Socket::SSL->new( + PeerHost => $strHost, PeerPort => 'https', SSL_verify_mode => $bVerifySsl ? SSL_VERIFY_PEER : SSL_VERIFY_NONE); + + if (!defined($oSocket)) + { + logErrorResult(ERROR_PROTOCOL, "unable to connect $!", $SSL_ERROR); + } + + # Create the buffered IO object + my $self = new pgBackRest::Common::Io::Buffered( + new pgBackRest::Common::Io::Handle('httpClient', $oSocket, $oSocket), $iProtocolTimeout, $lBufferMax); + + # Bless with the class + @ISA = $self->isA(); ## no critic (ClassHierarchies::ProhibitExplicitISA) + bless $self, $class; + + # Store socket + $self->{oSocket} = $oSocket; + + # Generate the query string + my $strQuery = httpQuery($hQuery); + + # Construct the request headers + $self->{strRequestHeader} = "${strVerb} ${strUri}?${strQuery} HTTP/1.1" . "\r\n"; + + foreach my $strHeader (sort(keys(%{$hRequestHeader}))) + { + $self->{strRequestHeader} .= "${strHeader}: $hRequestHeader->{$strHeader}\r\n"; + } + + $self->{strRequestHeader} .= "\r\n"; + + # Write request headers + $self->write(\$self->{strRequestHeader}); + + # Write content + if (defined($rstrRequestBody)) + { + my $iTotalSize = length($$rstrRequestBody); + my $iTotalSent = 0; + + # Write the request body in buffer-sized chunks + do + { + my $strBufferWrite = substr($$rstrRequestBody, $iTotalSent, $lBufferMax); + $iTotalSent += $self->write(\$strBufferWrite); + } while ($iTotalSent < $iTotalSize); + } + + # Read response code + ($self->{strResponseProtocol}, $self->{iResponseCode}, $self->{strResponseMessage}) = split(' ', trim($self->readLine())); + + # Read the response headers + $self->{iContentLength} = undef; + + my $strResponseHeader = ''; + my $strHeader = trim($self->readLine()); + + while ($strHeader ne '') + { + # Validate header + $strResponseHeader .= "${strHeader}\n"; + + my $iColonPos = index($strHeader, ':'); + + if ($iColonPos == -1) + { + confess &log(ERROR, "http header '${strHeader}' requires colon separator", ERROR_PROTOCOL); + } + + # Parse header + my $strHeaderKey = lc(substr($strHeader, 0, $iColonPos)); + my $strHeaderValue = trim(substr($strHeader, $iColonPos + 1)); + + # Store the header + $self->{hResponseHeader}{$strHeaderKey} = $strHeaderValue; + + # Process content length + if ($strHeaderKey eq HTTP_HEADER_CONTENT_LENGTH) + { + $self->{iContentLength} = $strHeaderValue + 0; + $self->{iContentRemaining} = $self->{iContentLength}; + } + # Process transfer encoding (only chunked is supported) + elsif ($strHeaderKey eq HTTP_HEADER_TRANSFER_ENCODING) + { + if ($strHeaderValue eq 'chunked') + { + $self->{iContentLength} = -1; + } + else + { + confess &log(ERROR, "invalid value '${strHeaderValue} for http header '${strHeaderKey}'", ERROR_PROTOCOL); + } + } + + # Read next header + $strHeader = trim($self->readLine()); + } + + # Test response code + if ($self->{iResponseCode} == 200) + { + # Content length should have been defined either by content-length or transfer encoding + if (!defined($self->{iContentLength})) + { + confess &log(ERROR, + HTTP_HEADER_CONTENT_LENGTH . ' or ' . HTTP_HEADER_TRANSFER_ENCODING . ' must be defined', ERROR_PROTOCOL); + } + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self} + ); +} + +#################################################################################################################################### +# read - read content from http stream +#################################################################################################################################### +sub read +{ + my $self = shift; + my $rtBuffer = shift; + my $iRequestSize = shift; + + # Make sure request size is not larger than what remains to be read + $iRequestSize = $iRequestSize < $self->{iContentRemaining} ? $iRequestSize : $self->{iContentRemaining}; + $self->{iContentRemaining} -= $iRequestSize; + + return $self->SUPER::read($rtBuffer, $iRequestSize, true); +} + +#################################################################################################################################### +# close/DESTROY - close the HTTP connection +#################################################################################################################################### +sub close +{ + my $self = shift; + + # Only close if the socket is open + if (defined($self->{oSocket})) + { + $self->{oSocket}->close(); + undef($self->{oSocket}); + } +} + +sub DESTROY {shift->close()} + +#################################################################################################################################### +# responseBody - return the entire body of the response in a buffer +#################################################################################################################################### +sub responseBody +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + ) = + logDebugParam + ( + __PACKAGE__ . '->responseBody' + ); + + my $strResponseBody = undef; + + # Nothing to do if content length is 0 + if ($self->{iContentLength} != 0) + { + while (1) + { + # Read chunk length + my $strChunkLength = trim($self->readLine()); + my $iChunkLength = hex($strChunkLength); + + # Exit if chunk length is 0 + last if ($iChunkLength == 0); + + # Read the chunk and consume the terminating LF + $self->SUPER::read(\$strResponseBody, $iChunkLength, true); + $self->readLine(); + }; + + $self->close(); + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'rstrResponseBody', value => \$strResponseBody, trace => true} + ); +} + +#################################################################################################################################### +# Properties. +#################################################################################################################################### +sub contentLength {shift->{iContentLength}} # Content length if available (-1 means not known yet) +sub responseCode {shift->{iResponseCode}} +sub responseHeader {shift->{hResponseHeader}} +sub responseMessage {shift->{strResponseMessage}} +sub responseProtocol {shift->{strResponseProtocol}} + +1; diff --git a/lib/pgBackRest/Common/Http/Common.pm b/lib/pgBackRest/Common/Http/Common.pm new file mode 100644 index 000000000..3ca4105f7 --- /dev/null +++ b/lib/pgBackRest/Common/Http/Common.pm @@ -0,0 +1,106 @@ +#################################################################################################################################### +# HTTP Common +#################################################################################################################################### +package pgBackRest::Common::Http::Common; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Exporter qw(import); + our @EXPORT = qw(); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; + +#################################################################################################################################### +# httpQuery - encode an HTTP query from a hash +#################################################################################################################################### +sub httpQuery +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $hQuery, + ) = + logDebugParam + ( + __PACKAGE__ . '::httpQuery', \@_, + {name => 'hQuery', required => false, trace => true}, + ); + + # Generate the query string + my $strQuery = ''; + + # If a hash (the normal case) + if (ref($hQuery)) + { + foreach my $strParam (sort(keys(%{$hQuery}))) + { + # Parameters may not be defined - this is OK + if (defined($hQuery->{$strParam})) + { + $strQuery .= ($strQuery eq '' ? '' : '&') . $strParam . '=' . httpUriEncode($hQuery->{$strParam}); + } + } + } + # Else query string was passed directly as a scalar + elsif (defined($hQuery)) + { + $strQuery = $hQuery; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strQuery', value => $strQuery, trace => true} + ); +} + +push @EXPORT, qw(httpQuery); + +#################################################################################################################################### +# httpUriEncode - encode query values to conform with URI specs +#################################################################################################################################### +sub httpUriEncode +{ + my $strString = shift; + + # Only encode if source string is defined + my $strEncodedString; + + if (defined($strString)) + { + # Iterate all characters in the string + for (my $iIndex = 0; $iIndex < length($strString); $iIndex++) + { + my $cChar = substr($strString, $iIndex, 1); + + # These characters are reproduced verbatim + if (($cChar ge 'A' && $cChar le 'Z') || ($cChar ge 'a' && $cChar le 'z') || ($cChar ge '0' && $cChar le '9') || + $cChar eq '_' || $cChar eq '-' || $cChar eq '~' || $cChar eq '.') + { + $strEncodedString .= $cChar; + } + # Forward slash is encoded + elsif ($cChar eq '/') + { + $strEncodedString .= '%2F'; + } + # All other characters are hex-encoded + else + { + $strEncodedString .= sprintf('%%%02X', ord($cChar)); + } + } + } + + return $strEncodedString; +} + +push @EXPORT, qw(httpUriEncode); + +1; diff --git a/lib/pgBackRest/Common/Xml.pm b/lib/pgBackRest/Common/Xml.pm new file mode 100644 index 000000000..e187dccd1 --- /dev/null +++ b/lib/pgBackRest/Common/Xml.pm @@ -0,0 +1,151 @@ +#################################################################################################################################### +# XML Helper Functions +#################################################################################################################################### +package pgBackRest::Common::Xml; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Exporter qw(import); + our @EXPORT = qw(); +use XML::LibXML; + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; + +#################################################################################################################################### +# xmlParse - parse a string into an xml document and return the root node +#################################################################################################################################### +use constant XML_HEADER => ''; + push @EXPORT, qw(XML_HEADER); + +#################################################################################################################################### +# xmlParse - parse a string into an xml document and return the root node +#################################################################################################################################### +sub xmlParse +{ + my $rstrXml = shift; + + my $oXml = XML::LibXML->load_xml(string => $rstrXml)->documentElement(); + + return $oXml; +} + +push @EXPORT, qw(xmlParse); + +#################################################################################################################################### +# xmlTagChildren - get all children that match the tag +#################################################################################################################################### +sub xmlTagChildren +{ + my $oXml = shift; + my $strTag = shift; + + return $oXml->getChildrenByTagName($strTag); +} + +push @EXPORT, qw(xmlTagChildren); + +#################################################################################################################################### +# xmlTagText - get the text content for a tag, error if the tag is required and does not exist +#################################################################################################################################### +sub xmlTagText +{ + my $oXml = shift; + my $strTag = shift; + my $bRequired = shift; + # my $strDefault = shift; + + # Get the tag or tags + my @oyTag = $oXml->getElementsByTagName($strTag); + + # Error if the tag does not exist and is required + if (@oyTag > 1) + { + confess &log(ERROR, @oyTag . " '${strTag}' tag(s) exist, but only one was expected", ERROR_FORMAT); + } + elsif (@oyTag == 0) + { + if (!defined($bRequired) || $bRequired) + { + confess &log(ERROR, "tag '${strTag}' does not exist", ERROR_FORMAT); + } + } + else + { + return $oyTag[0]->textContent(); + } + + return; +} + +push @EXPORT, qw(xmlTagText); + +#################################################################################################################################### +# xmlTagBool - get the boolean content for a tag, error if the tag is required and does not exist or is not boolean +#################################################################################################################################### +sub xmlTagBool +{ + my $oXml = shift; + my $strTag = shift; + my $bRequired = shift; + # my $strDefault = shift; + + # Test content for boolean value + my $strContent = xmlTagText($oXml, $strTag, $bRequired); + + if (defined($strContent)) + { + if ($strContent eq 'true') + { + return true; + } + elsif ($strContent eq 'false') + { + return false; + } + else + { + confess &log(ERROR, "invalid boolean value '${strContent}' for tag '${strTag}'", ERROR_FORMAT); + } + } + + return; +} + +push @EXPORT, qw(xmlTagBool); + +#################################################################################################################################### +# xmlTagInt - get the integer content for a tag, error if the tag is required and does not exist or is not an integer +#################################################################################################################################### +sub xmlTagInt +{ + my $oXml = shift; + my $strTag = shift; + my $bRequired = shift; + # my $strDefault = shift; + + # Test content for boolean value + my $iContent = xmlTagText($oXml, $strTag, $bRequired); + + if (defined($iContent)) + { + eval + { + $iContent = $iContent + 0; + return 1; + } + or do + { + confess &log(ERROR, "invalid integer value '${iContent}' for tag '${strTag}'", ERROR_FORMAT); + } + } + + return $iContent; +} + +push @EXPORT, qw(xmlTagInt); + +1; diff --git a/lib/pgBackRest/Config/Config.pm b/lib/pgBackRest/Config/Config.pm index 76a9f8a57..df78d5599 100644 --- a/lib/pgBackRest/Config/Config.pm +++ b/lib/pgBackRest/Config/Config.pm @@ -97,6 +97,8 @@ use constant REPO_TYPE_CIFS => 'cifs'; push @EXPORT, qw(REPO_TYPE_CIFS); use constant REPO_TYPE_POSIX => 'posix'; push @EXPORT, qw(REPO_TYPE_POSIX); +use constant REPO_TYPE_S3 => 's3'; + push @EXPORT, qw(REPO_TYPE_S3); #################################################################################################################################### # INFO Output Constants @@ -298,6 +300,22 @@ use constant OPTION_REPO_PATH => 'repo-pat use constant OPTION_REPO_TYPE => 'repo-type'; push @EXPORT, qw(OPTION_REPO_TYPE); +# Repository S3 +use constant OPTION_REPO_S3_KEY => 'repo-s3-key'; + push @EXPORT, qw(OPTION_REPO_S3_KEY); +use constant OPTION_REPO_S3_KEY_SECRET => 'repo-s3-key-secret'; + push @EXPORT, qw(OPTION_REPO_S3_KEY_SECRET); +use constant OPTION_REPO_S3_BUCKET => 'repo-s3-bucket'; + push @EXPORT, qw(OPTION_REPO_S3_BUCKET); +use constant OPTION_REPO_S3_ENDPOINT => 'repo-s3-endpoint'; + push @EXPORT, qw(OPTION_REPO_S3_ENDPOINT); +use constant OPTION_REPO_S3_HOST => 'repo-s3-host'; + push @EXPORT, qw(OPTION_REPO_S3_HOST); +use constant OPTION_REPO_S3_REGION => 'repo-s3-region'; + push @EXPORT, qw(OPTION_REPO_S3_REGION); +use constant OPTION_REPO_S3_VERIFY_SSL => 'repo-s3-verify-ssl'; + push @EXPORT, qw(OPTION_REPO_S3_VERIFY_SSL); + # Log level use constant OPTION_LOG_LEVEL_CONSOLE => 'log-level-console'; push @EXPORT, qw(OPTION_LOG_LEVEL_CONSOLE); @@ -486,6 +504,8 @@ use constant OPTION_DEFAULT_REPO_LINK => true; push @EXPORT, qw(OPTION_DEFAULT_REPO_LINK); use constant OPTION_DEFAULT_REPO_PATH => '/var/lib/' . BACKREST_EXE; push @EXPORT, qw(OPTION_DEFAULT_REPO_PATH); +use constant OPTION_DEFAULT_REPO_S3_VERIFY_SSL => true; + push @EXPORT, qw(OPTION_DEFAULT_REPO_S3_VERIFY_SSL); use constant OPTION_DEFAULT_REPO_TYPE => REPO_TYPE_POSIX; push @EXPORT, qw(OPTION_DEFAULT_REPO_TYPE); use constant OPTION_DEFAULT_SPOOL_PATH => '/var/spool/' . BACKREST_EXE; @@ -1271,6 +1291,53 @@ my %oOptionRule = }, }, + &OPTION_REPO_S3_BUCKET => + { + &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL, + &OPTION_RULE_DEPEND => + { + &OPTION_RULE_DEPEND_OPTION => OPTION_REPO_TYPE, + &OPTION_RULE_DEPEND_VALUE => REPO_TYPE_S3, + }, + &OPTION_RULE_COMMAND => OPTION_REPO_TYPE, + }, + + &OPTION_REPO_S3_KEY => + { + &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL, + &OPTION_RULE_SECURE => true, + &OPTION_RULE_REQUIRED => false, + &OPTION_RULE_DEPEND => + { + &OPTION_RULE_DEPEND_OPTION => OPTION_REPO_TYPE, + &OPTION_RULE_DEPEND_VALUE => REPO_TYPE_S3, + }, + &OPTION_RULE_COMMAND => OPTION_REPO_TYPE, + }, + + &OPTION_REPO_S3_KEY_SECRET => OPTION_REPO_S3_KEY, + + &OPTION_REPO_S3_ENDPOINT => OPTION_REPO_S3_BUCKET, + + &OPTION_REPO_S3_HOST => + { + &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL, + &OPTION_RULE_REQUIRED => false, + &OPTION_RULE_DEPEND => OPTION_REPO_S3_BUCKET, + &OPTION_RULE_COMMAND => OPTION_REPO_TYPE, + }, + + &OPTION_REPO_S3_REGION => OPTION_REPO_S3_BUCKET, + + &OPTION_REPO_S3_VERIFY_SSL => + { + &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL, + &OPTION_RULE_TYPE => OPTION_TYPE_BOOLEAN, + &OPTION_RULE_DEFAULT => OPTION_DEFAULT_REPO_S3_VERIFY_SSL, + &OPTION_RULE_COMMAND => OPTION_REPO_TYPE, + &OPTION_RULE_DEPEND => OPTION_REPO_S3_BUCKET, + }, + &OPTION_REPO_TYPE => { &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL, @@ -1280,6 +1347,7 @@ my %oOptionRule = { &REPO_TYPE_CIFS => true, &REPO_TYPE_POSIX => true, + &REPO_TYPE_S3 => true, }, &OPTION_RULE_COMMAND => { diff --git a/lib/pgBackRest/Config/ConfigHelpData.pm b/lib/pgBackRest/Config/ConfigHelpData.pm index 53595f12b..294f1dd38 100644 --- a/lib/pgBackRest/Config/ConfigHelpData.pm +++ b/lib/pgBackRest/Config/ConfigHelpData.pm @@ -608,6 +608,87 @@ my $oConfigHelpData = "need, though of course requirements will likely change over time as your database evolves." }, + # REPO-S3-BUCKET Option Help + #--------------------------------------------------------------------------------------------------------------------------- + 'repo-s3-bucket' => + { + section => 'general', + summary => + "S3 repository bucket.", + description => + "S3 bucket used to store the repository.\n" . + "\n" . + "pgBackRest repositories can be stored in the bucket root by setting repo-path=/ but it is usually best to " . + "specify a prefix, such as /repo, so logs and other AWS generated content can also be stored in the bucket." + }, + + # REPO-S3-ENDPOINT Option Help + #--------------------------------------------------------------------------------------------------------------------------- + 'repo-s3-endpoint' => + { + section => 'general', + summary => + "S3 repository endpoint.", + description => + "The AWS end point should be valid for the selected region." + }, + + # REPO-S3-HOST Option Help + #--------------------------------------------------------------------------------------------------------------------------- + 'repo-s3-host' => + { + section => 'general', + summary => + "S3 repository host.", + description => + "Connect to a host other than the end point. This is typically used for testing." + }, + + # REPO-S3-KEY Option Help + #--------------------------------------------------------------------------------------------------------------------------- + 'repo-s3-key' => + { + section => 'general', + summary => + "S3 repository access key.", + description => + "AWS key used to access this bucket." + }, + + # REPO-S3-KEY-SECRET Option Help + #--------------------------------------------------------------------------------------------------------------------------- + 'repo-s3-key-secret' => + { + section => 'general', + summary => + "S3 repository secret access key.", + description => + "AWS secret key used to access this bucket." + }, + + # REPO-S3-REGION Option Help + #--------------------------------------------------------------------------------------------------------------------------- + 'repo-s3-region' => + { + section => 'general', + summary => + "S3 repository region.", + description => + "The AWS region where the bucket was created." + }, + + # REPO-S3-VERIFY-SSL Option Help + #--------------------------------------------------------------------------------------------------------------------------- + 'repo-s3-verify-ssl' => + { + section => 'general', + summary => + "Verify S3 server certificate.", + description => + "Disables verification of the S3 server certificate. This should only be used for testing or other scenarios " . + "where a certificate has been self-signed." + }, + # REPO-TYPE Option Help #--------------------------------------------------------------------------------------------------------------------------- 'repo-type' => @@ -619,7 +700,8 @@ my $oConfigHelpData = "The following repository types are supported:\n" . "\n" . "* cifs - Like posix, but disables links and directory fsyncs\n" . - "* posix - Posix-compliant file systems" + "* posix - Posix-compliant file systems\n" . + "* s3 - AWS Simple Storage Service" }, # RESUME Option Help @@ -833,6 +915,13 @@ my $oConfigHelpData = 'neutral-umask' => 'section', 'protocol-timeout' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'stanza' => 'default' } @@ -875,6 +964,13 @@ my $oConfigHelpData = 'process-max' => 'section', 'protocol-timeout' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'spool-path' => 'section', 'stanza' => 'default' @@ -946,6 +1042,13 @@ my $oConfigHelpData = 'process-max' => 'section', 'protocol-timeout' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'resume' => 'section', 'retention-archive' => 'section', @@ -1023,6 +1126,13 @@ my $oConfigHelpData = 'online' => 'default', 'protocol-timeout' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'stanza' => 'default' } @@ -1054,6 +1164,13 @@ my $oConfigHelpData = 'log-path' => 'section', 'log-timestamp' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'retention-archive' => 'section', 'retention-archive-type' => 'section', @@ -1124,6 +1241,13 @@ my $oConfigHelpData = 'protocol-timeout' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'stanza' => 'default' } @@ -1188,6 +1312,13 @@ my $oConfigHelpData = 'protocol-timeout' => 'section', 'recovery-option' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', # SET Option Help @@ -1330,6 +1461,13 @@ my $oConfigHelpData = 'online' => 'default', 'protocol-timeout' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'stanza' => 'default' } @@ -1373,6 +1511,13 @@ my $oConfigHelpData = 'online' => 'default', 'protocol-timeout' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'stanza' => 'default' } @@ -1407,6 +1552,13 @@ my $oConfigHelpData = 'log-path' => 'section', 'log-timestamp' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'stanza' => 'default' } @@ -1457,6 +1609,13 @@ my $oConfigHelpData = 'log-path' => 'section', 'log-timestamp' => 'section', 'repo-path' => 'section', + 'repo-s3-bucket' => 'section', + 'repo-s3-endpoint' => 'section', + 'repo-s3-host' => 'section', + 'repo-s3-key' => 'section', + 'repo-s3-key-secret' => 'section', + 'repo-s3-region' => 'section', + 'repo-s3-verify-ssl' => 'section', 'repo-type' => 'section', 'stanza' => 'default' } diff --git a/lib/pgBackRest/Protocol/Storage/Helper.pm b/lib/pgBackRest/Protocol/Storage/Helper.pm index c2b299b80..ccfc088b8 100644 --- a/lib/pgBackRest/Protocol/Storage/Helper.pm +++ b/lib/pgBackRest/Protocol/Storage/Helper.pm @@ -184,7 +184,17 @@ sub storageRepo # Create the driver my $oDriver; - if (optionTest(OPTION_REPO_TYPE, REPO_TYPE_CIFS)) + if (optionTest(OPTION_REPO_TYPE, REPO_TYPE_S3)) + { + require pgBackRest::Storage::S3::Driver; + + $oDriver = new pgBackRest::Storage::S3::Driver( + optionGet(OPTION_REPO_S3_BUCKET), optionGet(OPTION_REPO_S3_ENDPOINT), optionGet(OPTION_REPO_S3_REGION), + optionGet(OPTION_REPO_S3_KEY), optionGet(OPTION_REPO_S3_KEY_SECRET), + {strHost => optionGet(OPTION_REPO_S3_HOST, false), bVerifySsl => optionGet(OPTION_REPO_S3_VERIFY_SSL, false)}, + lBufferMax => optionGet(OPTION_BUFFER_SIZE)); + } + elsif (optionTest(OPTION_REPO_TYPE, REPO_TYPE_CIFS)) { require pgBackRest::Storage::Cifs::Driver; diff --git a/lib/pgBackRest/Storage/S3/Auth.pm b/lib/pgBackRest/Storage/S3/Auth.pm new file mode 100644 index 000000000..ef7bb69be --- /dev/null +++ b/lib/pgBackRest/Storage/S3/Auth.pm @@ -0,0 +1,264 @@ +#################################################################################################################################### +# S3 Authentication +# +# Contains the functions required to do S3 authentication. It's a complicated topic and too much to cover here, but there is +# excellent documentation at http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html. +#################################################################################################################################### +package pgBackRest::Storage::S3::Auth; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Digest::SHA qw(hmac_sha256 hmac_sha256_hex sha256_hex); +use Exporter qw(import); + our @EXPORT = qw(); +use POSIX qw(strftime); + +use pgBackRest::Common::Log; + +#################################################################################################################################### +# Constants +#################################################################################################################################### +use constant S3 => 's3'; +use constant AWS4 => 'AWS4'; +use constant AWS4_REQUEST => 'aws4_request'; +use constant AWS4_HMAC_SHA256 => 'AWS4-HMAC-SHA256'; + +use constant S3_HEADER_AUTHORIZATION => 'authorization'; + push @EXPORT, qw(S3_HEADER_AUTHORIZATION); +use constant S3_HEADER_DATE => 'x-amz-date'; + push @EXPORT, qw(S3_HEADER_DATE); +use constant S3_HEADER_CONTENT_SHA256 => 'x-amz-content-sha256'; + push @EXPORT, qw(S3_HEADER_CONTENT_SHA256); +use constant S3_HEADER_HOST => 'host'; + push @EXPORT, qw(S3_HEADER_HOST); + +use constant PAYLOAD_DEFAULT_HASH => sha256_hex(''); + push @EXPORT, qw(PAYLOAD_DEFAULT_HASH); + +#################################################################################################################################### +# s3DateTime - format date/time for authentication +#################################################################################################################################### +sub s3DateTime +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $lTime, + ) = + logDebugParam + ( + __PACKAGE__ . '::s3DateTime', \@_, + {name => 'lTime', default => time(), trace => true}, + ); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strDateTime', value => strftime("%Y%m%dT%k%M%SZ", gmtime($lTime)), trace => true} + ); +} + +push @EXPORT, qw(s3DateTime); + +#################################################################################################################################### +# s3CanonicalRequest - strictly formatted version of the HTTP request used for signing +#################################################################################################################################### +sub s3CanonicalRequest +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strVerb, + $strUri, + $strQuery, + $hHeader, + $strPayloadHash, + ) = + logDebugParam + ( + __PACKAGE__ . '::s3CanonicalRequest', \@_, + {name => 'strVerb', trace => true}, + {name => 'strUri', trace => true}, + {name => 'strQuery', trace => true}, + {name => 'hHeader', trace => true}, + {name => 'strPayloadHash', trace => true}, + ); + + # Create the canonical request + my $strCanonicalRequest = + "${strVerb}\n${strUri}\n${strQuery}\n"; + my $strSignedHeaders; + + foreach my $strHeader (sort(keys(%{$hHeader}))) + { + if (lc($strHeader) ne $strHeader) + { + confess &log(ASSERT, "header '${strHeader}' must be lower case"); + } + + $strCanonicalRequest .= $strHeader . ":$hHeader->{$strHeader}\n"; + $strSignedHeaders .= (defined($strSignedHeaders) ? qw(;) : '') . lc($strHeader); + } + + $strCanonicalRequest .= "\n${strSignedHeaders}\n${strPayloadHash}"; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strCanonicalRequest', value => $strCanonicalRequest, trace => true}, + {name => 'strSignedHeaders', value => $strSignedHeaders, trace => true}, + ); +} + +push @EXPORT, qw(s3CanonicalRequest); + +#################################################################################################################################### +# s3SigningKey - signing keys last for seven days, but we'll regenerate every day because it doesn't seem too burdensome +#################################################################################################################################### +my $hSigningKeyCache; # Cache signing keys rather than regenerating them every time + +sub s3SigningKey +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strDate, + $strRegion, + $strSecretAccessKey, + ) = + logDebugParam + ( + __PACKAGE__ . '::s3SigningKey', \@_, + {name => 'strDate', trace => true}, + {name => 'strRegion', trace => true}, + {name => 'strSecretAccessKey', trace => true}, + ); + + # Check for signing key in cache + my $strSigningKey = $hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey}; + + # If not found then generate it + if (!defined($strSigningKey)) + { + my $strDateKey = hmac_sha256($strDate, AWS4 . $strSecretAccessKey); + my $strRegionKey = hmac_sha256($strRegion, $strDateKey); + my $strServiceKey = hmac_sha256(S3, $strRegionKey); + $strSigningKey = hmac_sha256(AWS4_REQUEST, $strServiceKey); + + # Cache the signing key + $hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey} = $strSigningKey; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strSigningKey', value => $strSigningKey, trace => true} + ); +} + +push @EXPORT, qw(s3SigningKey); + +#################################################################################################################################### +# s3StringToSign - string that will be signed by the signing key for authentication +#################################################################################################################################### +sub s3StringToSign +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strDateTime, + $strRegion, + $strCanonicalRequestHash, + ) = + logDebugParam + ( + __PACKAGE__ . '::s3StringToSign', \@_, + {name => 'strDateTime', trace => true}, + {name => 'strRegion', trace => true}, + {name => 'strCanonicalRequestHash', trace => true}, + ); + + my $strStringToSign = + AWS4_HMAC_SHA256 . "\n${strDateTime}\n" . substr($strDateTime, 0, 8) . "/${strRegion}/" . S3 . '/' . AWS4_REQUEST . "\n" . + $strCanonicalRequestHash; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'strStringToSign', value => $strStringToSign, trace => true} + ); +} + +push @EXPORT, qw(s3StringToSign); + +#################################################################################################################################### +# s3AuthorizationHeader - authorization string that will be used in the HTTP "authorization" header +#################################################################################################################################### +sub s3AuthorizationHeader +{ + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strRegion, + $strHost, + $strVerb, + $strUri, + $strQuery, + $strDateTime, + $hHeader, + $strAccessKeyId, + $strSecretAccessKey, + $strPayloadHash, + ) = + logDebugParam + ( + __PACKAGE__ . '::s3AuthorizationHeader', \@_, + {name => 'strRegion', trace => true}, + {name => 'strHost', trace => true}, + {name => 'strVerb', trace => true}, + {name => 'strUri', trace => true}, + {name => 'strQuery', trace => true}, + {name => 'strDateTime', trace => true}, + {name => 'hHeader', required => false, trace => true}, + {name => 'strAccessKeyId', trace => true}, + {name => 'strSecretAccessKey', trace => true}, + {name => 'strPayloadHash', trace => true}, + ); + + # Add s3 required headers + $hHeader->{&S3_HEADER_HOST} = $strHost; + $hHeader->{&S3_HEADER_CONTENT_SHA256} = $strPayloadHash; + $hHeader->{&S3_HEADER_DATE} = $strDateTime; + + # Create authorization string + my ($strCanonicalRequest, $strSignedHeaders) = s3CanonicalRequest($strVerb, $strUri, $strQuery, $hHeader, $strPayloadHash); + + $hHeader->{&S3_HEADER_AUTHORIZATION} = + AWS4_HMAC_SHA256 . " Credential=${strAccessKeyId}/" . substr($strDateTime, 0, 8) . "/${strRegion}/" . S3 . qw(/) . + AWS4_REQUEST . ",SignedHeaders=${strSignedHeaders},Signature=" . hmac_sha256_hex(s3StringToSign( + $strDateTime, $strRegion, sha256_hex($strCanonicalRequest)), + s3SigningKey(substr($strDateTime, 0, 8), $strRegion, $strSecretAccessKey)); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'hHeader', value => $hHeader, trace => true} + ); +} + +push @EXPORT, qw(s3AuthorizationHeader); + +1; diff --git a/lib/pgBackRest/Storage/S3/Driver.pm b/lib/pgBackRest/Storage/S3/Driver.pm new file mode 100644 index 000000000..90256a05f --- /dev/null +++ b/lib/pgBackRest/Storage/S3/Driver.pm @@ -0,0 +1,499 @@ +#################################################################################################################################### +# S3 Storage Driver +#################################################################################################################################### +package pgBackRest::Storage::S3::Driver; +use parent 'pgBackRest::Storage::S3::Request'; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Exporter qw(import); + our @EXPORT = qw(); +use Digest::MD5 qw(md5_base64); +use File::Basename qw(basename dirname); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::Common::Xml; +use pgBackRest::Storage::S3::FileRead; +use pgBackRest::Storage::S3::FileWrite; +use pgBackRest::Storage::S3::Request; +use pgBackRest::Storage::S3::Info; + +#################################################################################################################################### +# Package name constant +#################################################################################################################################### +use constant STORAGE_S3_DRIVER => __PACKAGE__; + push @EXPORT, qw(STORAGE_S3_DRIVER); + +#################################################################################################################################### +# Query constants +#################################################################################################################################### +use constant S3_QUERY_CONTINUATION_TOKEN => 'continuation-token'; +use constant S3_QUERY_DELIMITER => 'delimiter'; +use constant S3_QUERY_LIST_TYPE => 'list-type'; +use constant S3_QUERY_PREFIX => 'prefix'; + +#################################################################################################################################### +# Batch maximum size +#################################################################################################################################### +use constant S3_BATCH_MAX => 1000; + +#################################################################################################################################### +# openWrite - open a file for write +#################################################################################################################################### +sub openWrite +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFile, + ) = + logDebugParam + ( + __PACKAGE__ . '->openWrite', \@_, + {name => 'strFile', trace => true}, + ); + + my $oFileIO = new pgBackRest::Storage::S3::FileWrite($self, $strFile); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oFileIO', value => $oFileIO, trace => true}, + ); +} + +#################################################################################################################################### +# openRead +#################################################################################################################################### +sub openRead +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFile, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '->openRead', \@_, + {name => 'strFile', trace => true}, + {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, + ); + + my $oFileIO = new pgBackRest::Storage::S3::FileRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing}); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oFileIO', value => $oFileIO, trace => true}, + ); +} + +#################################################################################################################################### +# manifest +#################################################################################################################################### +sub manifest +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + $bRecurse, + $bPath, + ) = + logDebugParam + ( + __PACKAGE__ . '->manifest', \@_, + {name => 'strPath', trace => true}, + # Optional parameters not part of the driver spec + {name => 'bRecurse', optional => true, default => true, trace => true}, + {name => 'bPath', optional => true, default => true, trace => true}, + ); + + # Determine the prefix (this is the search path within the bucket) + my $strPrefix = $strPath eq qw{/} ? undef : substr($strPath, 1) . ($bPath ? qw{/} : ''); + + # A delimiter must be used if recursion is not desired + my $strDelimiter = $bRecurse ? undef : '/'; + + # Hash to hold the manifest + my $hManifest = {}; + + # Continuation token - returned from requests where there is more data to be fetched + my $strContinuationToken; + + do + { + # Get the file list + my $oResponse = $self->request( + HTTP_VERB_GET, {hQuery => + {&S3_QUERY_LIST_TYPE => 2, &S3_QUERY_PREFIX => $strPrefix, &S3_QUERY_DELIMITER => $strDelimiter, + &S3_QUERY_CONTINUATION_TOKEN => $strContinuationToken}, strResponseType => S3_RESPONSE_TYPE_XML}); + + # Modify the prefix for file searches so the filename is not stripped off + if (defined($strPrefix) && !$bPath) + { + # If there are no paths in the prefix then undef it + if (index($strPrefix, qw{/}) == -1) + { + undef($strPrefix); + } + else + { + $strPrefix = dirname($strPrefix) . qw{/}; + } + } + + # Store files + foreach my $oFile (xmlTagChildren($oResponse, "Contents")) + { + my $strName = xmlTagText($oFile, "Key"); + + # Strip off prefix + if (defined($strPrefix)) + { + $strName = substr($strName, length($strPrefix)); + } + + $hManifest->{$strName}->{type} = 'f'; + $hManifest->{$strName}->{size} = xmlTagText($oFile, "Size"); + + # Generate paths from the name if recursing + if ($bRecurse) + { + my @stryName = split(qw{/}, $strName); + + if (@stryName > 1) + { + $strName = undef; + + for (my $iIndex = 0; $iIndex < @stryName - 1; $iIndex++) + { + $strName .= (defined($strName) ? qw{/} : '') . $stryName[$iIndex]; + $hManifest->{$strName}->{type} = 'd'; + } + } + } + } + + # Store directories + if ($bPath && !$bRecurse) + { + foreach my $oPath (xmlTagChildren($oResponse, "CommonPrefixes")) + { + my $strName = xmlTagText($oPath, "Prefix"); + + # Strip off prefix + if (defined($strPrefix)) + { + $strName = substr($strName, length($strPrefix)); + } + + # Strip off final / + $strName = substr($strName, 0, length($strName) - 1); + + $hManifest->{$strName}->{type} = 'd'; + } + } + + $strContinuationToken = xmlTagText($oResponse, "NextContinuationToken", false); + } + while (defined($strContinuationToken)); + + # Add . for the initial path (this is just for compatibility with filesystems that have directories) + if ($bPath) + { + $hManifest->{qw{.}}->{type} = 'd'; + } + + # use Data::Dumper; &log(WARN, 'MANIFEST' . Dumper($hManifest)); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'hManifest', value => $hManifest, trace => true} + ); +} + +#################################################################################################################################### +# list - list a directory +#################################################################################################################################### +sub list +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + ) = + logDebugParam + ( + __PACKAGE__ . '->list', \@_, + {name => 'strPath', trace => true}, + ); + + # Get list using manifest function + my @stryFileList = grep(!/^\.$/i, keys(%{$self->manifest($strPath, {bRecurse => false})})); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'stryFileList', value => \@stryFileList, ref => true, trace => true} + ); +} + +#################################################################################################################################### +# pathCreate - directories do no exist in s3 so this is a noop +#################################################################################################################################### +sub pathCreate +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathCreate', \@_, + {name => 'strPath', trace => true}, + ); + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +#################################################################################################################################### +# pathSync - directories do not exist in s3 so this is a noop +#################################################################################################################################### +sub pathSync +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathSync', \@_, + {name => 'strPath', trace => true}, + ); + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +#################################################################################################################################### +# exists - check if a file exists +#################################################################################################################################### +sub exists +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFile, + ) = + logDebugParam + ( + __PACKAGE__ . '->exists', \@_, + {name => 'strFile', trace => true}, + ); + + # Does the path/file exist? + my $bExists = defined($self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)}) ? true : false; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'bExists', value => $bExists, trace => true} + ); +} + +#################################################################################################################################### +# pathExists +#################################################################################################################################### +sub pathExists +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strPath, + ) = + logDebugParam + ( + __PACKAGE__ . '->pathExists', \@_, + {name => 'strPath', trace => true}, + ); + + my $bExists = true; + + # Only check if path <> / + if ($strPath ne qw{/}) + { + # Does the path exist? + my $rhInfo = $self->manifest(dirname($strPath), {bRecurse => false, bPath => true})->{basename($strPath)}; + $bExists = defined($rhInfo) && $rhInfo->{type} eq 'd' ? true : false; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'bExists', value => $bExists, trace => true} + ); +} + +#################################################################################################################################### +# info - get information about a file +#################################################################################################################################### +sub info +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strFile, + ) = + logDebugParam + ( + __PACKAGE__ . '->info', \@_, + {name => 'strFile', trace => true}, + ); + + # Get the file + my $rhFile = $self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)}; + + if (!defined($rhFile)) + { + confess &log(ERROR, "unable to get info for missing file ${strFile}", ERROR_FILE_MISSING); + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oInfo', value => new pgBackRest::Storage::S3::Info($rhFile->{size}), trace => true} + ); +} + +#################################################################################################################################### +# remove +#################################################################################################################################### +sub remove +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $rstryFile, + $bRecurse, + ) = + logDebugParam + ( + __PACKAGE__ . '->remove', \@_, + {name => 'rstryFile', trace => true}, + {name => 'bRecurse', optional => true, default => false, trace => true}, + ); + + # Remove a tree + if ($bRecurse) + { + my $rhManifest = $self->manifest($rstryFile); + my @stryRemoveFile; + + # Iterate all files in the manifest + foreach my $strFile (sort({$b cmp $a} keys(%{$rhManifest}))) + { + next if $rhManifest->{$strFile}->{type} eq 'd'; + push(@stryRemoveFile, "${rstryFile}/${strFile}"); + } + + # Remove files + if (@stryRemoveFile > 0) + { + $self->remove(\@stryRemoveFile); + } + } + # Only remove the specified file + else + { + # If stryFile is a scalar, convert to an array + my $rstryFileAll = ref($rstryFile) ? $rstryFile : [$rstryFile]; + + do + { + my $strFile = shift(@{$rstryFileAll}); + my $iTotal = 0; + my $strXml = XML_HEADER . 'true'; + + while (defined($strFile)) + { + $iTotal++; + $strXml .= '' . substr($strFile, 1) . ''; + + $strFile = $iTotal < 2 ? shift(@{$rstryFileAll}) : undef; + } + + $strXml .= ''; + + my $hHeader = {'content-md5' => md5_base64($strXml) . '=='}; + + # Delete a file + my $oResponse = $self->request( + HTTP_VERB_POST, + {hQuery => 'delete=', rstrBody => \$strXml, hHeader => $hHeader, strResponseType => S3_RESPONSE_TYPE_XML}); + } + while (@{$rstryFileAll} > 0); + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'bResult', value => true, trace => true} + ); +} + + +#################################################################################################################################### +# Getters +#################################################################################################################################### +sub capability {false} +sub className {STORAGE_S3_DRIVER} + +1; diff --git a/lib/pgBackRest/Storage/S3/FileRead.pm b/lib/pgBackRest/Storage/S3/FileRead.pm new file mode 100644 index 000000000..79505b8eb --- /dev/null +++ b/lib/pgBackRest/Storage/S3/FileRead.pm @@ -0,0 +1,66 @@ +#################################################################################################################################### +# S3 File Read +#################################################################################################################################### +package pgBackRest::Storage::S3::FileRead; +use parent 'pgBackRest::Common::Io::Base'; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Digest::MD5 qw(md5_base64); +use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC); +use File::Basename qw(dirname); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::Common::Xml; +use pgBackRest::Storage::Base; +use pgBackRest::Storage::S3::Request; + +#################################################################################################################################### +# CONSTRUCTOR +#################################################################################################################################### +our @ISA = (); ## no critic (ClassHierarchies::ProhibitExplicitISA) + +sub new +{ + my $class = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $oDriver, + $strName, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'oDriver', trace => true}, + {name => 'strName', trace => true}, + {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, + ); + + # Open file + my $self = $oDriver->request( + HTTP_VERB_GET, {strUri => $strName, strResponseType => S3_RESPONSE_TYPE_IO, bIgnoreMissing => $bIgnoreMissing}); + + # Bless with new class if file exists + if (defined($self)) + { + @ISA = $self->isA(); ## no critic (ClassHierarchies::ProhibitExplicitISA) + bless $self, $class; + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self, trace => true} + ); +} + +1; diff --git a/lib/pgBackRest/Storage/S3/FileWrite.pm b/lib/pgBackRest/Storage/S3/FileWrite.pm new file mode 100644 index 000000000..9241c08c5 --- /dev/null +++ b/lib/pgBackRest/Storage/S3/FileWrite.pm @@ -0,0 +1,194 @@ +#################################################################################################################################### +# S3 File Write +#################################################################################################################################### +package pgBackRest::Storage::S3::FileWrite; +use parent 'pgBackRest::Common::Io::Base'; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Digest::MD5 qw(md5_base64); +use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC); +use File::Basename qw(dirname); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::Common::Xml; +use pgBackRest::Storage::Base; +use pgBackRest::Storage::S3::Request; + +#################################################################################################################################### +# Constants +#################################################################################################################################### +use constant S3_BUFFER_MAX => 16777216; + +#################################################################################################################################### +# CONSTRUCTOR +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $oDriver, + $strName, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'oDriver', trace => true}, + {name => 'strName', trace => true}, + ); + + # Create the class hash + my $self = $class->SUPER::new("'${strName}'"); + bless $self, $class; + + # Set variables + $self->{oDriver} = $oDriver; + $self->{strName} = $strName; + + # Start with an empty buffer + $self->{rtBuffer} = ''; + + # Has anything been written? + $self->{bWritten} = false; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self, trace => true} + ); +} + +#################################################################################################################################### +# open - open the file +#################################################################################################################################### +sub open +{ + my $self = shift; + + # Request an upload id + my $oResponse = $self->{oDriver}->request( + HTTP_VERB_POST, {strUri => $self->{strName}, hQuery => 'uploads=', strResponseType => S3_RESPONSE_TYPE_XML}); + + $self->{strUploadId} = xmlTagText($oResponse, 'UploadId'); + + # Intialize the multi-part array + $self->{rstryMultiPart} = []; +} + +#################################################################################################################################### +# write - write data to a file +#################################################################################################################################### +sub write +{ + my $self = shift; + my $rtBuffer = shift; + + # Note that write has been called + $self->{bWritten} = true; + + if (defined($rtBuffer)) + { + $self->{rtBuffer} .= $$rtBuffer; + + # Wait until buffer is at least max before writing to avoid writing smaller files multi-part + if (length($self->{rtBuffer}) >= S3_BUFFER_MAX) + { + $self->flush(); + } + + return length($$rtBuffer); + } + + return 0; +} + +#################################################################################################################################### +# flush - flush whatever is in the buffer out +#################################################################################################################################### +sub flush +{ + my $self = shift; + + # Open file if it is not open already + $self->open() if !$self->opened(); + + # Put a file + $self->{oDriver}->request( + HTTP_VERB_PUT, + {strUri => $self->{strName}, + hQuery => {'partNumber' => @{$self->{rstryMultiPart}} + 1, 'uploadId' => $self->{strUploadId}}, + rstrBody => \$self->{rtBuffer}, hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}}); + + # Store the returned etag + push(@{$self->{rstryMultiPart}}, $self->{oDriver}->{hResponseHeader}{&S3_HEADER_ETAG}); + + # Clear the buffer + $self->{rtBuffer} = ''; +} + +#################################################################################################################################### +# close - close the file +#################################################################################################################################### +sub close +{ + my $self = shift; + + # Only close if something was written + if ($self->{bWritten}) + { + # Make sure close does not run again + $self->{bWritten} = false; + + # If the file is open then multipart transfer has already started and must be completed + if ($self->opened()) + { + # flush out whatever is in the buffer + $self->flush(); + + my $strXml = XML_HEADER . ''; + my $iPartNo = 0; + + foreach my $strETag (@{$self->{rstryMultiPart}}) + { + $iPartNo++; + + $strXml .= "${iPartNo}${strETag}"; + } + + $strXml .= ''; + + # Finalize file + my $oResponse = $self->{oDriver}->request( + HTTP_VERB_POST, + {strUri => $self->{strName}, hQuery => {'uploadId' => $self->{strUploadId}}, + rstrBody => \$strXml, hHeader => {'content-md5' => md5_base64($strXml) . '=='}, + strResponseType => S3_RESPONSE_TYPE_XML}); + } + # Else the file can be transmitted in one block + else + { + $self->{oDriver}->request( + HTTP_VERB_PUT, + {strUri => $self->{strName}, rstrBody => \$self->{rtBuffer}, + hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}}); + } + } + + return true; +} + +#################################################################################################################################### +# Getters +#################################################################################################################################### +sub opened {defined(shift->{strUploadId})} + +1; diff --git a/lib/pgBackRest/Storage/S3/Info.pm b/lib/pgBackRest/Storage/S3/Info.pm new file mode 100644 index 000000000..865d1cfbb --- /dev/null +++ b/lib/pgBackRest/Storage/S3/Info.pm @@ -0,0 +1,48 @@ +#################################################################################################################################### +# S3 File Info +#################################################################################################################################### +package pgBackRest::Storage::S3::Info; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use pgBackRest::Common::Log; + +#################################################################################################################################### +# new +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Create the class hash + my $self = {}; + bless $self, $class; + + # Assign function parameters, defaults, and log debug info + ( + my $strOperation, + $self->{lSize}, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'lSize'}, + ); + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self} + ); +} + +#################################################################################################################################### +# Getters +#################################################################################################################################### +sub size {shift->{lSize}} + +1; diff --git a/lib/pgBackRest/Storage/S3/Request.pm b/lib/pgBackRest/Storage/S3/Request.pm new file mode 100644 index 000000000..9bcf0eba6 --- /dev/null +++ b/lib/pgBackRest/Storage/S3/Request.pm @@ -0,0 +1,202 @@ +#################################################################################################################################### +# S3 Request +#################################################################################################################################### +package pgBackRest::Storage::S3::Request; + +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Digest::SHA qw(hmac_sha256 hmac_sha256_hex sha256_hex); +use Exporter qw(import); + our @EXPORT = qw(); +use IO::Socket::SSL; + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Http::Client; +use pgBackRest::Common::Http::Common; +use pgBackRest::Common::Io::Base; +use pgBackRest::Common::Log; +use pgBackRest::Common::String; +use pgBackRest::Common::Xml; +use pgBackRest::Storage::S3::Auth; + +#################################################################################################################################### +# Constants +#################################################################################################################################### +use constant HTTP_VERB_GET => 'GET'; + push @EXPORT, qw(HTTP_VERB_GET); +use constant HTTP_VERB_POST => 'POST'; + push @EXPORT, qw(HTTP_VERB_POST); +use constant HTTP_VERB_PUT => 'PUT'; + push @EXPORT, qw(HTTP_VERB_PUT); + +use constant S3_HEADER_CONTENT_LENGTH => 'content-length'; + push @EXPORT, qw(S3_HEADER_CONTENT_LENGTH); +use constant S3_HEADER_TRANSFER_ENCODING => 'transfer-encoding'; + push @EXPORT, qw(S3_HEADER_TRANSFER_ENCODING); +use constant S3_HEADER_ETAG => 'etag'; + push @EXPORT, qw(S3_HEADER_ETAG); + +use constant S3_RESPONSE_TYPE_IO => 'io'; + push @EXPORT, qw(S3_RESPONSE_TYPE_IO); +use constant S3_RESPONSE_TYPE_NONE => 'none'; + push @EXPORT, qw(S3_RESPONSE_TYPE_NONE); +use constant S3_RESPONSE_TYPE_XML => 'xml'; + push @EXPORT, qw(S3_RESPONSE_TYPE_XML); + +#################################################################################################################################### +# new +#################################################################################################################################### +sub new +{ + my $class = shift; + + # Create the class hash + my $self = {}; + bless $self, $class; + + # Assign function parameters, defaults, and log debug info + ( + my $strOperation, + $self->{strBucket}, + $self->{strEndPoint}, + $self->{strRegion}, + $self->{strAccessKeyId}, + $self->{strSecretAccessKey}, + $self->{strHost}, + $self->{bVerifySsl}, + $self->{lBufferMax}, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + {name => 'strBucket', trace => true}, + {name => 'strEndPoint', trace => true}, + {name => 'strRegion', trace => true}, + {name => 'strAccessKeyId', trace => true}, + {name => 'strSecretAccessKey', trace => true}, + {name => 'strHost', optional => true, trace => true}, + {name => 'bVerifySsl', optional => true, default => true, trace => true}, + {name => 'lBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX, trace => true}, + ); + + # If host is not set then it will be bucket + endpoint + $self->{strHost} = defined($self->{strHost}) ? $self->{strHost} : "$self->{strBucket}.$self->{strEndPoint}"; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self, trace => true} + ); +} + +#################################################################################################################################### +# request - send a request to S3 +#################################################################################################################################### +sub request +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strVerb, + $strUri, + $hQuery, + $hHeader, + $rstrBody, + $strResponseType, + $bIgnoreMissing, + ) = + logDebugParam + ( + __PACKAGE__ . '->request', \@_, + {name => 'strVerb', trace => true}, + {name => 'strUri', optional => true, default => '/', trace => true}, + {name => 'hQuery', optional => true, trace => true}, + {name => 'hHeader', optional => true, trace => true}, + {name => 'rstrBody', optional => true, trace => true}, + {name => 'strResponseType', optional => true, default => S3_RESPONSE_TYPE_NONE, trace => true}, + {name => 'bIgnoreMissing', optional => true, default => false, trace => true}, + ); + + # Get datetime to be used for auth requests + my $strDateTime = s3DateTime(); + + # Set content length and hash + $hHeader->{&S3_HEADER_CONTENT_SHA256} = defined($rstrBody) ? sha256_hex($$rstrBody) : PAYLOAD_DEFAULT_HASH; + $hHeader->{&S3_HEADER_CONTENT_LENGTH} = defined($rstrBody) ? length($$rstrBody) : 0; + + # Generate authorization header + $hHeader = s3AuthorizationHeader( + $self->{strRegion}, "$self->{strBucket}.$self->{strEndPoint}", $strVerb, $strUri, httpQuery($hQuery), $strDateTime, + $hHeader, $self->{strAccessKeyId}, $self->{strSecretAccessKey}, $hHeader->{&S3_HEADER_CONTENT_SHA256}); + + # Send the request + my $oHttpClient = new pgBackRest::Common::Http::Client( + $self->{strHost}, $strVerb, + {strUri => $strUri, hQuery => $hQuery, hRequestHeader => $hHeader, rstrRequestBody => $rstrBody, + bVerifySsl => $self->{bVerifySsl}, lBufferMax => $self->{lBufferMax}}); + + # Check response code + my $iReponseCode = $oHttpClient->responseCode(); + my $oResponse; + + if ($iReponseCode == 200) + { + # Save the response headers locally + $self->{hResponseHeader} = $oHttpClient->responseHeader(); + + # XML response is expected + if ($strResponseType eq S3_RESPONSE_TYPE_XML) + { + my $rtResponseBody = $oHttpClient->responseBody(); + + if ($oHttpClient->contentLength() == 0 || !defined($$rtResponseBody)) + { + confess &log(ERROR, + "response type '${strResponseType}' was requested but content length is zero or content is missing", + ERROR_PROTOCOL); + } + + $oResponse = xmlParse($$rtResponseBody); + } + # An IO object is expected for file responses + elsif ($strResponseType eq S3_RESPONSE_TYPE_IO) + { + $oResponse = $oHttpClient; + } + } + else + { + if ($iReponseCode == 404) + { + if (!$bIgnoreMissing) + { + confess &log(ERROR, "unable to open '${strUri}': No such file or directory", ERROR_FILE_MISSING); + } + } + else + { + my $rstrResponseBody = $oHttpClient->responseBody(); + + confess &log(ERROR, + "S3 request error [$iReponseCode] " . $oHttpClient->responseMessage() . + (defined($$rstrResponseBody) ? ":\n${$rstrResponseBody}" : ''), + ERROR_PROTOCOL); + } + } + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'oResponse', value => $oResponse, trace => true, ref => true} + ); +} + +1; diff --git a/test/Vagrantfile b/test/Vagrantfile index 4defe91ec..c61ab6801 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -8,7 +8,7 @@ Vagrant.configure(2) do |config| config.vm.box_version = "20170603.0.0" # vagrant plugin install vagrant-disksize - # config.disksize.size = '64GB' + config.disksize.size = '64GB' config.vm.provider :virtualbox do |vb| vb.name = "pgbackrest-test" @@ -37,12 +37,29 @@ Vagrant.configure(2) do |config| #--------------------------------------------------------------------------------------------------------------------------- echo 'Install Perl Modules' && date - apt-get install -y libdbd-pg-perl libxml-checker-perl libperl-critic-perl libdevel-nytprof-perl + apt-get install -y libdbd-pg-perl libio-socket-ssl-perl libxml-libxml-perl libxml-checker-perl libperl-critic-perl \ + libdevel-nytprof-perl #--------------------------------------------------------------------------------------------------------------------------- echo 'Install Build Tools' && date apt-get install -y devscripts build-essential lintian git txt2man debhelper + #--------------------------------------------------------------------------------------------------------------------------- + echo 'Install AWS CLI' && date + apt-get install -y python-pip + pip install --upgrade awscli + + # Configure AWS CLI + sudo -i -u ubuntu aws configure set region us-east-1 + sudo -i -u ubuntu aws configure set aws_access_key_id accessKey1 + sudo -i -u ubuntu aws configure set aws_secret_access_key verySecretKey1 + + # Create test alias for AWS CLI + echo '' >> /home/ubuntu/.profile + echo '# Test alias for AWS CLI' >> /home/ubuntu/.profile + echo 'alias s3-test="export PYTHONWARNINGS=ignore && aws s3 --endpoint-url=https://172.17.0.2 --no-verify-ssl"' \ + >> /home/ubuntu/.profile + #--------------------------------------------------------------------------------------------------------------------------- echo 'Build Devel::Cover' && date apt-get install -y libpod-coverage-perl libtest-differences-perl libhtml-parser-perl libtemplate-perl diff --git a/test/expect/archive-get-001.log b/test/expect/archive-get-001.log index 4d1607207..e4a7a4fee 100644 --- a/test/expect/archive-get-001.log +++ b/test/expect/archive-get-001.log @@ -1,5 +1,5 @@ -run 001 - rmt 0, cmp 0, exists 0 -================================ +run 001 - rmt 0, cmp 0, exists 0, s3 0 +====================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-get-002.log b/test/expect/archive-get-002.log index 6ae699c79..4799e73b9 100644 --- a/test/expect/archive-get-002.log +++ b/test/expect/archive-get-002.log @@ -1,5 +1,5 @@ -run 002 - rmt 0, cmp 0, exists 1 -================================ +run 002 - rmt 0, cmp 0, exists 1, s3 0 +====================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-get-003.log b/test/expect/archive-get-003.log index be1ba6b82..2767598ed 100644 --- a/test/expect/archive-get-003.log +++ b/test/expect/archive-get-003.log @@ -1,5 +1,5 @@ -run 003 - rmt 0, cmp 1, exists 0 -================================ +run 003 - rmt 0, cmp 1, exists 0, s3 0 +====================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-get-004.log b/test/expect/archive-get-004.log index 3d1905c98..979279318 100644 --- a/test/expect/archive-get-004.log +++ b/test/expect/archive-get-004.log @@ -1,5 +1,5 @@ -run 004 - rmt 0, cmp 1, exists 1 -================================ +run 004 - rmt 0, cmp 1, exists 1, s3 0 +====================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-get-005.log b/test/expect/archive-get-005.log index c1f24e462..c86278da3 100644 --- a/test/expect/archive-get-005.log +++ b/test/expect/archive-get-005.log @@ -1,5 +1,5 @@ -run 005 - rmt 1, cmp 0, exists 0 -================================ +run 005 - rmt 1, cmp 0, exists 0, s3 0 +====================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-get-006.log b/test/expect/archive-get-006.log index 08168bd30..8a4ca002c 100644 --- a/test/expect/archive-get-006.log +++ b/test/expect/archive-get-006.log @@ -1,5 +1,5 @@ -run 006 - rmt 1, cmp 0, exists 1 -================================ +run 006 - rmt 1, cmp 0, exists 1, s3 0 +====================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-get-007.log b/test/expect/archive-get-007.log index 8ba65b9fb..6f87bbb61 100644 --- a/test/expect/archive-get-007.log +++ b/test/expect/archive-get-007.log @@ -1,5 +1,5 @@ -run 007 - rmt 1, cmp 1, exists 0 -================================ +run 007 - rmt 1, cmp 1, exists 0, s3 0 +====================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-get-008.log b/test/expect/archive-get-008.log index d2a203301..3e9d4a537 100644 --- a/test/expect/archive-get-008.log +++ b/test/expect/archive-get-008.log @@ -1,5 +1,5 @@ -run 008 - rmt 1, cmp 1, exists 1 -================================ +run 008 - rmt 1, cmp 1, exists 1, s3 0 +====================================== > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-get-009.log b/test/expect/archive-get-009.log new file mode 100644 index 000000000..d5ac47c26 --- /dev/null +++ b/test/expect/archive-get-009.log @@ -0,0 +1,181 @@ +run 009 - rmt 1, cmp 1, exists 1, s3 1 +====================================== + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db +P00 INFO: get WAL segment 000000010000000100000001 +P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef] +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 ERROR: [055]: raised on 'backup remote' host: archive.info does not exist but is required to push/get WAL segments + HINT: is archive_command configured in postgresql.conf? + HINT: has a stanza-create been performed? + HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme. +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = [undef], oException = [object], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = false, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: archive-get command end: aborted with exception [055] +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 55 + +stanza-create db - create required data for stanza (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-2] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get --cmd-ssh=/usr/bin/ssh 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db +P00 INFO: get WAL segment 000000010000000100000001 +P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = /usr/bin/ssh, strHost = backup, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = /usr/bin/ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef] +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strArchiveId = 9.4-1 +P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /9.4-1/000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] +P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0 +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: archive-get command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000002 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db +P00 INFO: get WAL segment 000000010000000100000002 +P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002, strSourceArchive = 000000010000000100000002 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000002, ullDbSysId = [undef] +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strArchiveId = 9.4-1 +P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /9.4-1/000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] +P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0 +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: archive-get command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000003 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db +P00 INFO: get WAL segment 000000010000000100000003 +P00 DEBUG: Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003, strSourceArchive = 000000010000000100000003 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000003, ullDbSysId = [undef] +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strArchiveId = 9.4-1 +P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /9.4-1/000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003 +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] +P00 DEBUG: Archive::ArchiveGet->get=>: iResult = 0 +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: archive-get command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 diff --git a/test/expect/archive-push-001.log b/test/expect/archive-push-001.log index 1f4fe070d..c8be77094 100644 --- a/test/expect/archive-push-001.log +++ b/test/expect/archive-push-001.log @@ -1,5 +1,5 @@ -run 001 - rmt 0, cmp 0, arc_async 0 -=================================== +run 001 - rmt 0, cmp 0, arc_async 0, s3 0 +========================================= > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-push-002.log b/test/expect/archive-push-002.log index fc5c418ff..d50e7177f 100644 --- a/test/expect/archive-push-002.log +++ b/test/expect/archive-push-002.log @@ -1,5 +1,5 @@ -run 002 - rmt 0, cmp 0, arc_async 1 -=================================== +run 002 - rmt 0, cmp 0, arc_async 1, s3 0 +========================================= > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-push-003.log b/test/expect/archive-push-003.log index 40f34bbf1..008d30b40 100644 --- a/test/expect/archive-push-003.log +++ b/test/expect/archive-push-003.log @@ -1,5 +1,5 @@ -run 003 - rmt 0, cmp 1, arc_async 0 -=================================== +run 003 - rmt 0, cmp 1, arc_async 0, s3 0 +========================================= > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-push-004.log b/test/expect/archive-push-004.log index 8b9622227..14241a9db 100644 --- a/test/expect/archive-push-004.log +++ b/test/expect/archive-push-004.log @@ -1,5 +1,5 @@ -run 004 - rmt 0, cmp 1, arc_async 1 -=================================== +run 004 - rmt 0, cmp 1, arc_async 1, s3 0 +========================================= > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-push-005.log b/test/expect/archive-push-005.log index 1d273fde0..da9f3ccf2 100644 --- a/test/expect/archive-push-005.log +++ b/test/expect/archive-push-005.log @@ -1,5 +1,5 @@ -run 005 - rmt 1, cmp 0, arc_async 0 -=================================== +run 005 - rmt 1, cmp 0, arc_async 0, s3 0 +========================================= > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-push-006.log b/test/expect/archive-push-006.log index a4e2d36d8..2aabb352d 100644 --- a/test/expect/archive-push-006.log +++ b/test/expect/archive-push-006.log @@ -1,5 +1,5 @@ -run 006 - rmt 1, cmp 0, arc_async 1 -=================================== +run 006 - rmt 1, cmp 0, arc_async 1, s3 0 +========================================= > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-push-007.log b/test/expect/archive-push-007.log index fdc56e8b2..35bdadde6 100644 --- a/test/expect/archive-push-007.log +++ b/test/expect/archive-push-007.log @@ -1,5 +1,5 @@ -run 007 - rmt 1, cmp 1, arc_async 0 -=================================== +run 007 - rmt 1, cmp 1, arc_async 0, s3 0 +========================================= > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-push-008.log b/test/expect/archive-push-008.log index c306b45a9..1135058dd 100644 --- a/test/expect/archive-push-008.log +++ b/test/expect/archive-push-008.log @@ -1,5 +1,5 @@ -run 008 - rmt 1, cmp 1, arc_async 1 -=================================== +run 008 - rmt 1, cmp 1, arc_async 1, s3 0 +========================================= > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24 ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/archive-push-009.log b/test/expect/archive-push-009.log new file mode 100644 index 000000000..87a11af5b --- /dev/null +++ b/test/expect/archive-push-009.log @@ -0,0 +1,263 @@ +run 009 - rmt 1, cmp 1, arc_async 1, s3 1 +========================================= + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --archive-max-mb=24 --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 WARN: 'archive-max-mb' is no longer not longer valid, use 'archive-queue-max' instead +P00 ERROR: [055]: raised on 'local-1' host: raised on 'backup remote' host: archive.info does not exist but is required to push/get WAL segments + HINT: is archive_command configured in postgresql.conf? + HINT: has a stanza-create been performed? + HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme. +P00 INFO: archive-push command end: aborted with exception [055] + +stanza-create db - create required data for stanza (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-2] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push --cmd-ssh=/usr/bin/ssh [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000001 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment version 9.4 does not match archive version 8.0 + HINT: are you archiving to the correct stanza? +P00 INFO: archive-push command end: aborted with exception [044] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment system-id 6353949018581704918 does not match archive system-id 5000900090001855000 + HINT: are you archiving to the correct stanza? +P00 INFO: archive-push command end: aborted with exception [044] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 WARN: WAL segment 000000010000000100000001 already exists in the archive with the same checksum + HINT: this is valid in some recovery scenarios but may also indicate a problem. +P00 INFO: pushed WAL segment 000000010000000100000001 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000001 already exists in the archive +P00 INFO: archive-push command end: aborted with exception [045] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001.partial +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000001.partial asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001.partial +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 WARN: WAL segment 000000010000000100000001.partial already exists in the archive with the same checksum + HINT: this is valid in some recovery scenarios but may also indicate a problem. +P00 INFO: pushed WAL segment 000000010000000100000001.partial asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001.partial +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000001.partial already exists in the archive +P00 INFO: archive-push command end: aborted with exception [045] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000002 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000003 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000004 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000004 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push --cmd-ssh=/usr/bin/ssh [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000005 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment version 9.4 does not match archive version 8.0 + HINT: are you archiving to the correct stanza? +P00 INFO: archive-push command end: aborted with exception [044] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment system-id 6353949018581704918 does not match archive system-id 5000900090001855000 + HINT: are you archiving to the correct stanza? +P00 INFO: archive-push command end: aborted with exception [044] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 WARN: WAL segment 000000010000000100000005 already exists in the archive with the same checksum + HINT: this is valid in some recovery scenarios but may also indicate a problem. +P00 INFO: pushed WAL segment 000000010000000100000005 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000005 already exists in the archive +P00 INFO: archive-push command end: aborted with exception [045] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005.partial +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000005.partial asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005.partial +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 WARN: WAL segment 000000010000000100000005.partial already exists in the archive with the same checksum + HINT: this is valid in some recovery scenarios but may also indicate a problem. +P00 INFO: pushed WAL segment 000000010000000100000005.partial asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005.partial +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000005.partial already exists in the archive +P00 INFO: archive-push command end: aborted with exception [045] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000006 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000006 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000007 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000007 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000008 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000008 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push --cmd-ssh=/usr/bin/ssh [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000009 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment version 9.4 does not match archive version 8.0 + HINT: are you archiving to the correct stanza? +P00 INFO: archive-push command end: aborted with exception [044] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment system-id 6353949018581704918 does not match archive system-id 5000900090001855000 + HINT: are you archiving to the correct stanza? +P00 INFO: archive-push command end: aborted with exception [044] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 WARN: WAL segment 000000010000000100000009 already exists in the archive with the same checksum + HINT: this is valid in some recovery scenarios but may also indicate a problem. +P00 INFO: pushed WAL segment 000000010000000100000009 asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000009 already exists in the archive +P00 INFO: archive-push command end: aborted with exception [045] + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009.partial +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: pushed WAL segment 000000010000000100000009.partial asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009.partial +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 WARN: WAL segment 000000010000000100000009.partial already exists in the archive with the same checksum + HINT: this is valid in some recovery scenarios but may also indicate a problem. +P00 INFO: pushed WAL segment 000000010000000100000009.partial asynchronously +P00 INFO: archive-push command end: completed successfully + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009.partial +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000009.partial already exists in the archive +P00 INFO: archive-push command end: aborted with exception [045] + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} diff --git a/test/expect/archive-stop-001.log b/test/expect/archive-stop-001.log index 6efceff49..2dc6aff76 100644 --- a/test/expect/archive-stop-001.log +++ b/test/expect/archive-stop-001.log @@ -1,5 +1,5 @@ -run 001 - rmt 0, cmp 0, error version -===================================== +run 001 - rmt 0, cmp 0, error version, s3 0 +=========================================== stanza-create db - create required data for stanza (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create diff --git a/test/expect/archive-stop-002.log b/test/expect/archive-stop-002.log index 855c125dd..9147fc9ef 100644 --- a/test/expect/archive-stop-002.log +++ b/test/expect/archive-stop-002.log @@ -1,5 +1,5 @@ -run 002 - rmt 0, cmp 1, error version -===================================== +run 002 - rmt 0, cmp 1, error version, s3 0 +=========================================== stanza-create db - create required data for stanza (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create diff --git a/test/expect/archive-stop-003.log b/test/expect/archive-stop-003.log index 21678fdd0..6afdc14a8 100644 --- a/test/expect/archive-stop-003.log +++ b/test/expect/archive-stop-003.log @@ -1,5 +1,5 @@ -run 003 - rmt 1, cmp 0, error version -===================================== +run 003 - rmt 1, cmp 0, error version, s3 0 +=========================================== stanza-create db - create required data for stanza (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create diff --git a/test/expect/archive-stop-004.log b/test/expect/archive-stop-004.log index 88917534a..552499e14 100644 --- a/test/expect/archive-stop-004.log +++ b/test/expect/archive-stop-004.log @@ -1,5 +1,5 @@ -run 004 - rmt 1, cmp 0, error connect -===================================== +run 004 - rmt 1, cmp 0, error connect, s3 0 +=========================================== stanza-create db - create required data for stanza (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create diff --git a/test/expect/archive-stop-005.log b/test/expect/archive-stop-005.log index 06e7f8a6d..590800870 100644 --- a/test/expect/archive-stop-005.log +++ b/test/expect/archive-stop-005.log @@ -1,5 +1,5 @@ -run 005 - rmt 1, cmp 1, error version -===================================== +run 005 - rmt 1, cmp 1, error version, s3 0 +=========================================== stanza-create db - create required data for stanza (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create diff --git a/test/expect/archive-stop-006.log b/test/expect/archive-stop-006.log index 9974fbdf1..f180c8144 100644 --- a/test/expect/archive-stop-006.log +++ b/test/expect/archive-stop-006.log @@ -1,5 +1,5 @@ -run 006 - rmt 1, cmp 1, error connect -===================================== +run 006 - rmt 1, cmp 1, error connect, s3 0 +=========================================== stanza-create db - create required data for stanza (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create diff --git a/test/expect/archive-stop-007.log b/test/expect/archive-stop-007.log new file mode 100644 index 000000000..25ce733d3 --- /dev/null +++ b/test/expect/archive-stop-007.log @@ -0,0 +1,58 @@ +run 007 - rmt 1, cmp 0, error connect, s3 1 +=========================================== + +stanza-create db - create required data for stanza (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db --backup-host=bogus archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 +------------------------------------------------------------------------------------------------------------------------------------ +P00 ERROR: [042]: process 'bogus remote' terminated unexpectedly: ssh: Could not resolve hostname bogus: Name or service not known + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db --backup-host=bogus archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003 +------------------------------------------------------------------------------------------------------------------------------------ +P00 ERROR: [042]: process 'bogus remote' terminated unexpectedly: ssh: Could not resolve hostname bogus: Name or service not known + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000004 +------------------------------------------------------------------------------------------------------------------------------------ +P00 WARN: dropped WAL file 000000010000000100000004 because archive queue exceeded 33554432 bytes + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005 +------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/full-real-001.log b/test/expect/full-real-001.log index efaec7f72..c8d5022a3 100644 --- a/test/expect/full-real-001.log +++ b/test/expect/full-real-001.log @@ -1,5 +1,5 @@ -run 001 - bkp 0, sby 0, dst db-master, asy 0, cmp 0 -=================================================== +run 001 - bkp 0, sby 0, dst db-master, asy 0, cmp 0, s3 0 +========================================================= stanza-create db - main create stanza info files (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-002.log b/test/expect/full-real-002.log index 29cce7597..1a069753a 100644 --- a/test/expect/full-real-002.log +++ b/test/expect/full-real-002.log @@ -1,5 +1,5 @@ -run 002 - bkp 0, sby 0, dst db-master, asy 0, cmp 1 -=================================================== +run 002 - bkp 0, sby 0, dst db-master, asy 0, cmp 1, s3 0 +========================================================= stanza-create db - main create stanza info files (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-003.log b/test/expect/full-real-003.log index 28bc9efec..3d1dc2979 100644 --- a/test/expect/full-real-003.log +++ b/test/expect/full-real-003.log @@ -1,5 +1,5 @@ -run 003 - bkp 0, sby 0, dst db-master, asy 1, cmp 0 -=================================================== +run 003 - bkp 0, sby 0, dst db-master, asy 1, cmp 0, s3 0 +========================================================= stanza-create db - main create stanza info files (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-004.log b/test/expect/full-real-004.log index 7b0acfd86..843621b14 100644 --- a/test/expect/full-real-004.log +++ b/test/expect/full-real-004.log @@ -1,5 +1,5 @@ -run 004 - bkp 0, sby 0, dst db-master, asy 1, cmp 1 -=================================================== +run 004 - bkp 0, sby 0, dst db-master, asy 1, cmp 1, s3 0 +========================================================= stanza-create db - main create stanza info files (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-005.log b/test/expect/full-real-005.log index a94fee8a0..99dfedccd 100644 --- a/test/expect/full-real-005.log +++ b/test/expect/full-real-005.log @@ -1,5 +1,5 @@ -run 005 - bkp 0, sby 1, dst db-master, asy 0, cmp 0 -=================================================== +run 005 - bkp 0, sby 1, dst db-master, asy 0, cmp 0, s3 0 +========================================================= stanza-create db - main create stanza info files (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-006.log b/test/expect/full-real-006.log index adce4ed91..0aa9db248 100644 --- a/test/expect/full-real-006.log +++ b/test/expect/full-real-006.log @@ -1,5 +1,5 @@ -run 006 - bkp 0, sby 1, dst db-standby, asy 0, cmp 0 -==================================================== +run 006 - bkp 0, sby 1, dst db-standby, asy 0, cmp 0, s3 0 +========================================================== stanza-create db - main create stanza info files (db-standby host) > [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-007.log b/test/expect/full-real-007.log index 3501da158..2fb810517 100644 --- a/test/expect/full-real-007.log +++ b/test/expect/full-real-007.log @@ -1,5 +1,5 @@ -run 007 - bkp 1, sby 0, dst backup, asy 0, cmp 0 -================================================ +run 007 - bkp 1, sby 0, dst backup, asy 0, cmp 0, s3 0 +====================================================== stanza-create db - main create stanza info files (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-008.log b/test/expect/full-real-008.log index 5821a137f..0149b13ea 100644 --- a/test/expect/full-real-008.log +++ b/test/expect/full-real-008.log @@ -1,5 +1,5 @@ -run 008 - bkp 1, sby 0, dst backup, asy 0, cmp 1 -================================================ +run 008 - bkp 1, sby 0, dst backup, asy 0, cmp 1, s3 0 +====================================================== stanza-create db - main create stanza info files (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-009.log b/test/expect/full-real-009.log index 2530ab424..022d1e47b 100644 --- a/test/expect/full-real-009.log +++ b/test/expect/full-real-009.log @@ -1,5 +1,5 @@ -run 009 - bkp 1, sby 0, dst backup, asy 1, cmp 0 -================================================ +run 009 - bkp 1, sby 0, dst backup, asy 1, cmp 0, s3 0 +====================================================== stanza-create db - main create stanza info files (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-010.log b/test/expect/full-real-010.log index e78fa16e8..506ac13bb 100644 --- a/test/expect/full-real-010.log +++ b/test/expect/full-real-010.log @@ -1,5 +1,5 @@ -run 010 - bkp 1, sby 0, dst backup, asy 1, cmp 1 -================================================ +run 010 - bkp 1, sby 0, dst backup, asy 1, cmp 1, s3 0 +====================================================== stanza-create db - main create stanza info files (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-011.log b/test/expect/full-real-011.log index bdd3a318d..cf9e0aa52 100644 --- a/test/expect/full-real-011.log +++ b/test/expect/full-real-011.log @@ -1,5 +1,5 @@ -run 011 - bkp 1, sby 1, dst backup, asy 0, cmp 0 -================================================ +run 011 - bkp 1, sby 1, dst backup, asy 0, cmp 0, s3 0 +====================================================== stanza-create db - main create stanza info files (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create diff --git a/test/expect/full-real-012.log b/test/expect/full-real-012.log new file mode 100644 index 000000000..8f419e8ee --- /dev/null +++ b/test/expect/full-real-012.log @@ -0,0 +1,387 @@ +run 012 - bkp 1, sby 0, dst backup, asy 1, cmp 1, s3 1 +====================================================== + +stanza-create db - main create stanza info files (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on missing archive.info file (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +stanza-create db - force create stanza info files (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ + +full backup - fail on archive_mode=off (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on archive_mode=off (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on archive_mode=off (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +full backup - fail on invalid archive_command (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on invalid archive_command (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on invalid archive_command (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on archive timeout when archive-check=n (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --no-archive-check --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - verify success (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - verify success (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on archive mismatch after upgrade (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on archive mismatch after upgrade (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on archive timeout (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on archive timeout (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on backup info mismatch (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - fail on backup info mismatch (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - verify success after backup (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +check db - verify success after backup (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + +stanza-create db - verify success with force (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ + +stanza-create db - fail on database mismatch with directory (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --db-path=[TEST_PATH]/db-master/db/testbase/ stanza-create +------------------------------------------------------------------------------------------------------------------------------------ + +stanza-create db - successfully create stanza files to be upgraded (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --db-path=[TEST_PATH]/db-master/db/testbase/ --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ + +stanza-upgrade db - upgrade stanza files online (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-upgrade +------------------------------------------------------------------------------------------------------------------------------------ + +full backup - fail on backup lock exists (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ + +full backup - update during backup (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --buffer-size=16384 --type=full --stanza=db backup --test --test-delay=1 --test-point=manifest-build=y +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base +db-port=[PORT-1] +db-socket-path=[TEST_PATH]/db-master/db + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-1] +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 +spool-path=[TEST_PATH]/db-master/spool + +[global:archive-push] +archive-async=y + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-port=[PORT-1] +db-user=[USER-2] + +[global] +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + +stop all stanzas (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf stop +------------------------------------------------------------------------------------------------------------------------------------ + +incr backup - attempt backup when stopped (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ + +start all stanzas (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf start +------------------------------------------------------------------------------------------------------------------------------------ + +incr backup - fail on archive_mode=always (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ + +incr backup - fail on backup already running (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ + +incr backup - update during backup (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stop-auto --no-archive-check --buffer-size=32768 --stanza=db backup --test --test-delay=1 --test-point=manifest-build=y +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base +db-port=[PORT-1] +db-socket-path=[TEST_PATH]/db-master/db + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-1] +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 +spool-path=[TEST_PATH]/db-master/spool + +[global:archive-push] +archive-async=y + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-port=[PORT-1] +db-user=[USER-2] + +[global] +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + +restore, type 'default', expect exit 38 - postmaster running (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --link-all --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + +restore, type 'default', expect exit 40 - path not empty (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --link-all --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + +restore, type 'default' (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-include=test1 --link-all --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' + +restore, force, backup '[BACKUP-FULL-1]', type 'immediate' (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --force --set=[BACKUP-FULL-1] --type=immediate --link-all --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' +recovery_target = 'immediate' + +restore, force, backup '[BACKUP-INCR-1]', type 'xid', target '[XID-TARGET-1]', target-action=promote (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --force --set=[BACKUP-INCR-1] --tablespace-map-all=../../tablespace --type=xid --target="[XID-TARGET-1]" --link-all --target-action=promote --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' +recovery_target_xid = '[XID-TARGET-1]' +recovery_target_action = 'promote' + +restore, type 'preserve' (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --type=preserve --link-all --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' +recovery_target_xid = '[XID-TARGET-1]' +recovery_target_action = 'promote' + +restore delta, backup '[BACKUP-FULL-1]', type 'time', target '[TIMESTAMP-TARGET-1]' (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-FULL-1] --type=time --target="[TIMESTAMP-TARGET-1]" --link-all --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' +recovery_target_time = '[TIMESTAMP-TARGET-1]' + +restore delta, backup '[BACKUP-INCR-1]', type 'xid', target '[XID-TARGET-1]', exclusive (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-INCR-1] --type=xid --target="[XID-TARGET-1]" --target-exclusive --link-all --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' +recovery_target_xid = '[XID-TARGET-1]' +recovery_target_inclusive = 'false' + +restore delta, force, type 'name', target 'backrest' (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --force --type=name --target="backrest" --link-all --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' +recovery_target_name = 'backrest' + +restore delta, backup '[BACKUP-INCR-1]', type 'default', timeline '4' (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-INCR-1] --target-timeline="4" --link-all --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +standby_mode = 'on' +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' +recovery_target_timeline = '4' + +incr backup - fail on --no-online (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ + +incr backup - succeed on --no-online with --force (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --force --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base +db-port=[PORT-1] +db-socket-path=[TEST_PATH]/db-master/db +recovery-option=standby-mode=on + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-1] +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 +spool-path=[TEST_PATH]/db-master/spool + +[global:archive-push] +archive-async=y + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-port=[PORT-1] +db-user=[USER-2] + +[global] +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y diff --git a/test/expect/full-synthetic-001.log b/test/expect/full-synthetic-001.log index 8a1ad7feb..2c3cd1d3a 100644 --- a/test/expect/full-synthetic-001.log +++ b/test/expect/full-synthetic-001.log @@ -1,5 +1,5 @@ -run 001 - rmt 0, cmp 0, hardlink 0 -================================== +run 001 - rmt 0, cmp 0, hardlink 0, s3 0 +======================================== info all stanzas - no stanzas exist (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info diff --git a/test/expect/full-synthetic-002.log b/test/expect/full-synthetic-002.log index 74c55fa7d..88f314e62 100644 --- a/test/expect/full-synthetic-002.log +++ b/test/expect/full-synthetic-002.log @@ -1,5 +1,5 @@ -run 002 - rmt 0, cmp 0, hardlink 1 -================================== +run 002 - rmt 0, cmp 0, hardlink 1, s3 0 +======================================== info all stanzas - no stanzas exist (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info diff --git a/test/expect/full-synthetic-003.log b/test/expect/full-synthetic-003.log index 826a77c3e..0ebe3887a 100644 --- a/test/expect/full-synthetic-003.log +++ b/test/expect/full-synthetic-003.log @@ -1,5 +1,5 @@ -run 003 - rmt 0, cmp 1, hardlink 0 -================================== +run 003 - rmt 0, cmp 1, hardlink 0, s3 0 +======================================== info all stanzas - no stanzas exist (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info diff --git a/test/expect/full-synthetic-004.log b/test/expect/full-synthetic-004.log index 08d41b45b..0836d8793 100644 --- a/test/expect/full-synthetic-004.log +++ b/test/expect/full-synthetic-004.log @@ -1,5 +1,5 @@ -run 004 - rmt 0, cmp 1, hardlink 1 -================================== +run 004 - rmt 0, cmp 1, hardlink 1, s3 0 +======================================== info all stanzas - no stanzas exist (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info diff --git a/test/expect/full-synthetic-005.log b/test/expect/full-synthetic-005.log index 83fe6c8d2..7fcfb8330 100644 --- a/test/expect/full-synthetic-005.log +++ b/test/expect/full-synthetic-005.log @@ -1,5 +1,5 @@ -run 005 - rmt 1, cmp 0, hardlink 0 -================================== +run 005 - rmt 1, cmp 0, hardlink 0, s3 0 +======================================== info all stanzas - no stanzas exist (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info diff --git a/test/expect/full-synthetic-006.log b/test/expect/full-synthetic-006.log index fa43a3471..0f06518d1 100644 --- a/test/expect/full-synthetic-006.log +++ b/test/expect/full-synthetic-006.log @@ -1,5 +1,5 @@ -run 006 - rmt 1, cmp 0, hardlink 1 -================================== +run 006 - rmt 1, cmp 0, hardlink 1, s3 0 +======================================== info all stanzas - no stanzas exist (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info diff --git a/test/expect/full-synthetic-007.log b/test/expect/full-synthetic-007.log index b8bb817ee..b5b9f6f3c 100644 --- a/test/expect/full-synthetic-007.log +++ b/test/expect/full-synthetic-007.log @@ -1,5 +1,5 @@ -run 007 - rmt 1, cmp 1, hardlink 0 -================================== +run 007 - rmt 1, cmp 1, hardlink 0, s3 0 +======================================== info all stanzas - no stanzas exist (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info diff --git a/test/expect/full-synthetic-008.log b/test/expect/full-synthetic-008.log index 3e69f63cd..92a5c0a7f 100644 --- a/test/expect/full-synthetic-008.log +++ b/test/expect/full-synthetic-008.log @@ -1,5 +1,5 @@ -run 008 - rmt 1, cmp 1, hardlink 1 -================================== +run 008 - rmt 1, cmp 1, hardlink 1, s3 0 +======================================== info all stanzas - no stanzas exist (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info diff --git a/test/expect/full-synthetic-009.log b/test/expect/full-synthetic-009.log new file mode 100644 index 000000000..12b4b38a9 --- /dev/null +++ b/test/expect/full-synthetic-009.log @@ -0,0 +1,5075 @@ +run 009 - rmt 1, cmp 0, hardlink 0, s3 1 +======================================== + +info all stanzas - no stanzas exist (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info +------------------------------------------------------------------------------------------------------------------------------------ +No stanzas exist in /. + +info all stanzas - no stanzas exist (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --output=json info +------------------------------------------------------------------------------------------------------------------------------------ +[] + +stanza-create db - create required data for stanza (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +full backup - error on identical link destinations (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 ERROR: [070]: link [TEST_PATH]/db-master/db/base/postgresql.conf (../pg_config) references a subdirectory of or the same directory as link [TEST_PATH]/db-master/db/base/pg_config_bad (../../db/pg_config) +P00 INFO: backup command end: aborted with exception [070] + +full backup - error on link to a link (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 ERROR: [070]: link '[TEST_PATH]/db-master/db/base/postgresql.conf.bad' -> '../pg_config/postgresql.conf.link' cannot reference another link +P00 INFO: backup command end: aborted with exception [070] + +full backup - create pg_stat link, pg_clog dir (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --manifest-save-threshold=3 --protocol-timeout=2 --db-timeout=1 --cmd-ssh=/usr/bin/ssh --buffer-size=16384 --checksum-page --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=16384 --checksum-page --cmd-ssh=/usr/bin/ssh --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=1 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --manifest-save-threshold=3 --no-online --protocol-timeout=2 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/backup/log +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 16384, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = <0750>, strPathExp = /backup.history +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = () +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 16384, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 2, strCommand = [BACKREST-BIN] --buffer-size=16384 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=1 --protocol-timeout=2 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote, strCommandSSH = /usr/bin/ssh, strHost = db-master, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 16384, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 2, strCommand = /usr/bin/ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no [USER-1]@db-master '[BACKREST-BIN] --buffer-size=16384 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=1 --protocol-timeout=2 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote', strId = 'db-master remote', strName = remote +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check(): bRequired = , iCatalogVersion = 201409291, iControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check=>: iDbHistoryId = 1 +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = () +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[BACKUP-FULL-1], strPathExp = , strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = () +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^[BACKUP-FULL-1], strPathExp = /backup.history/[YEAR-1], strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = () +P00 DEBUG: Protocol::Storage::Remote->exists(): strPathExp = [TEST_PATH]/db-master/db/base/postmaster.pid +P00 DEBUG: Protocol::Storage::Remote->exists=>: bExists = false +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = [undef], hDatabaseMap = [undef], hTablespaceMap = [undef], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = [undef], strParentPath = [undef], strPath = [TEST_PATH]/db-master/db/base +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base/pg_tblspc +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/pg_stat, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_stat +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/postgresql.conf, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_config/postgresql.conf +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_config/postgresql.conf +P00 DEBUG: Backup::Backup->process: create backup path /backup/db/[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = , strMode = <0750>, strPathExp = /[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::Backup->processManifest(): bCompress = false, bHardLink = false, oBackupManifest = [object], strBackupLabel = [BACKUP-FULL-1], strDbCopyPath = [TEST_PATH]/db-master/db/base, strDbMasterPath = [TEST_PATH]/db-master/db/base, strDbVersion = 9.4, strLsnStart = [undef], strType = full +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Local::Process->new(): bConfessError = , iSelectTimeout = <1>, strBackRestBin = <[BACKREST-BIN]>, strHostType = db +P00 DEBUG: Protocol::Local::Process->hostAdd(): iHostConfigIdx = 1, iProcessMax = 1 +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-1]/pg_data +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-1]/pg_data/base +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-1]/pg_data/base/1 +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-1]/pg_data/base/16384 +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-1]/pg_data/base/32768 +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-1]/pg_data/global +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-1]/pg_data/pg_clog +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-1]/pg_data/pg_stat +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-1]/pg_data/pg_tblspc +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/33001, pg_data/base/32768/33001, 65536, [undef], 1, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/32768/33001, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/33000.32767, pg_data/base/32768/33000.32767, 32768, [undef], 1, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/32768/33000.32767, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/33000, pg_data/base/32768/33000, 32768, [undef], 1, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/32768/33000, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/16384/17000, pg_data/base/16384/17000, 16384, [undef], 1, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/16384/17000, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/global/pg_control, pg_data/global/pg_control, 8192, [undef], 0, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-2], 0, [undef]), strKey = pg_data/global/pg_control, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/1/12000, pg_data/base/1/12000, 8192, [undef], 1, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/1/12000, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/postgresql.conf, pg_data/postgresql.conf, 21, [undef], 0, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-2], 1, [undef]), strKey = pg_data/postgresql.conf, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/pg_stat/global.stat, pg_data/pg_stat/global.stat, 5, [undef], 0, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-2], 1, [undef]), strKey = pg_data/pg_stat/global.stat, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/PG_VERSION, pg_data/base/32768/PG_VERSION, 3, [undef], 0, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/base/32768/PG_VERSION, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/16384/PG_VERSION, pg_data/base/16384/PG_VERSION, 3, [undef], 0, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/base/16384/PG_VERSION, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/1/PG_VERSION, pg_data/base/1/PG_VERSION, 3, [undef], 0, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/base/1/PG_VERSION, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/PG_VERSION, pg_data/PG_VERSION, 3, [undef], 0, [BACKUP-FULL-1], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/PG_VERSION, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->hostConnect: start local process: iHostConfigIdx = 1, iHostIdx = 0, iHostProcessIdx = 0, iProcessId = 1, strHostType = db +P00 DEBUG: Protocol::Local::Master->new(): iProcessIdx = 1, strCommand = [BACKREST-BIN] --buffer-size=16384 --cmd-ssh=/usr/bin/ssh --command=backup --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=1 --db-user=[USER-1] --host-id=1 --lock-path=[TEST_PATH]/backup/lock --log-path=[TEST_PATH]/backup/log --process=1 --protocol-timeout=2 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db local +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 16384, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 2, strCommand = [BACKREST-BIN] --buffer-size=16384 --cmd-ssh=/usr/bin/ssh --command=backup --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=1 --db-user=[USER-1] --host-id=1 --lock-path=[TEST_PATH]/backup/lock --log-path=[TEST_PATH]/backup/log --process=1 --protocol-timeout=2 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db local, strId = 'local-1', strName = local +P00 DEBUG: Protocol::Local::Process->hostConnect=>: bResult = true +P00 DEBUG: Protocol::Local::Process->init: init local process: iDirection = 1, iHostIdx = 0, iProcessId = 1, iQueueIdx = 0, iQueueLastIdx = 0 +P00 DEBUG: Protocol::Local::Process->init=>: bResult = true +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/33001, strQueueIdx = 0 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 65536, 65536, 6bf316f11d28c28914ea9be92c00de9bea6d9a6b, {bAlign => 1, bValid => 0, iyPageError => (0, (3, 5), 7)}), strKey = pg_data/base/32768/33001 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/33000.32767, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/base/32768/33001 (64KB, 39%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P00 WARN: invalid page checksums found in file db-master:[TEST_PATH]/db-master/db/base/base/32768/33001 at pages 0, 3-5, 7 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 65536, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 32768, 32768, 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5, {bAlign => 1, bValid => 1}), strKey = pg_data/base/32768/33000.32767 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/33000, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/base/32768/33000.32767 (32KB, 59%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 32768, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 32768, 32768, 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f, {bAlign => 1, bValid => 1}), strKey = pg_data/base/32768/33000 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/16384/17000, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/base/32768/33000 (32KB, 79%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 32768, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 16384, 16384, e0101dd8ffb910c9c202ca35b5f828bcb9697bed, {bAlign => 1, bValid => 0, iyPageError => (1)}), strKey = pg_data/base/16384/17000 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/global/pg_control, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/base/16384/17000 (16KB, 89%) checksum e0101dd8ffb910c9c202ca35b5f828bcb9697bed +P00 WARN: invalid page checksum found in file db-master:[TEST_PATH]/db-master/db/base/base/16384/17000 at page 1 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 16384, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 8192, 8192, 89373d9f2973502940de06bc5212489df3f8a912, [undef]), strKey = pg_data/global/pg_control +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/1/12000, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/global/pg_control (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 8192, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 8192, 8192, 22c98d248ff548311eda88559e4a8405ed77c003, {bAlign => 1, bValid => 1}), strKey = pg_data/base/1/12000 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/postgresql.conf, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/base/1/12000 (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 8192, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 21, 21, 6721d92c9fcdf4248acff1f9a1377127d9064807, [undef]), strKey = pg_data/postgresql.conf +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/pg_stat/global.stat, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/postgresql.conf (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 21, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 5, 5, e350d5ce0153f3e22d5db21cf2a4eff00f3ee877, [undef]), strKey = pg_data/pg_stat/global.stat +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/PG_VERSION, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_stat/global.stat (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 5, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 3, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, [undef]), strKey = pg_data/base/32768/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/16384/PG_VERSION, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/base/32768/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 3, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 3, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, [undef]), strKey = pg_data/base/16384/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/1/PG_VERSION, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/base/16384/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 3, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 3, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, [undef]), strKey = pg_data/base/1/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/PG_VERSION, strQueueIdx = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/base/1/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 3, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 3, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, [undef]), strKey = pg_data/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: no jobs found, stop local: iHostConfigIdx = [undef], iHostIdx = 0, iProcessId = 1, strHostType = [undef] +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/PG_VERSION (3B, 100%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Backup::File::backupManifestUpdate: save manifest: lManifestSaveCurrent = 3, lManifestSaveSize = 3 +P00 DEBUG: Protocol::Local::Process->process: all jobs complete +P00 DEBUG: Backup::Backup->processManifest=>: lSizeTotal = 163878 +P00 INFO: full backup size = 160KB +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = db +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Storage::Local->pathSync(): bRecurse = true, strPathExp = /[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db/[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db/[BACKUP-FULL-1] +P00 INFO: new backup label = [BACKUP-FULL-1] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = true, bPathCreate = true, lTimestamp = [undef], rhyFilter = ({strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup.history/[YEAR-1]/[BACKUP-FULL-1].manifest.gz +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = /[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup.history +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = /latest +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/backup.info.copy +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --buffer-size=16384 --cmd-ssh=/usr/bin/ssh --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=1 --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=2 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-1]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-1] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-1]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-1] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: expire command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-FULL-1]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-FULL-1]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="full" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base","type":"path"} +pg_data/pg_stat={"path":"../pg_stat","type":"link"} +pg_data/postgresql.conf={"file":"postgresql.conf","path":"../pg_config","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"e0101dd8ffb910c9c202ca35b5f828bcb9697bed","checksum-page":false,"checksum-page-error":[1],"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-2]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_stat={"destination":"../pg_stat"} +pg_data/postgresql.conf={"destination":"../pg_config/postgresql.conf"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-1]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +full backup - protocol timeout (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --protocol-timeout=1 --db-timeout=.1 --type=full --stanza=db backup --test --test-delay=1 --test-point=backup-start=y +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=.1 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=1 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --test --test-delay=1 --test-point=backup-start=y --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/backup/log +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = <0750>, strPathExp = /backup.history +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-1]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-1] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=.1 --protocol-timeout=1 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote, strCommandSSH = ssh, strHost = db-master, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 1, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no [USER-1]@db-master '[BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=.1 --protocol-timeout=1 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote', strId = 'db-master remote', strName = remote +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check(): bRequired = , iCatalogVersion = 201409291, iControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check=>: iDbHistoryId = 1 +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-1]) +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-1]/backup.manifest.copy +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[BACKUP-FULL-2], strPathExp = , strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = () +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^[BACKUP-FULL-2], strPathExp = /backup.history/[YEAR-1], strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = () +P00 DEBUG: Protocol::Storage::Remote->exists(): strPathExp = [TEST_PATH]/db-master/db/base/postmaster.pid +P00 DEBUG: Protocol::Storage::Remote->exists=>: bExists = false +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = [undef], hDatabaseMap = [undef], hTablespaceMap = [undef], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = [undef], strParentPath = [undef], strPath = [TEST_PATH]/db-master/db/base +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base/pg_tblspc +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/pg_stat, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_stat +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/postgresql.conf, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_config/postgresql.conf +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_config/postgresql.conf +P00 DEBUG: Backup::Backup->process: create backup path /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = , strMode = <0750>, strPathExp = /[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Backup::Backup->processManifest(): bCompress = false, bHardLink = false, oBackupManifest = [object], strBackupLabel = [BACKUP-FULL-2], strDbCopyPath = [TEST_PATH]/db-master/db/base, strDbMasterPath = [TEST_PATH]/db-master/db/base, strDbVersion = 9.4, strLsnStart = [undef], strType = full +P00 TEST: PgBaCkReStTeSt-BACKUP-START-PgBaCkReStTeSt +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 ERROR: [042]: process 'db-master remote' terminated unexpectedly [042]: unable to read line after 1 second(s) from stdio + +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = [undef], oException = [object], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = false, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: backup command end: aborted with exception [042] +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 42 + +stop all stanzas (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --force stop +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stop command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --config=[TEST_PATH]/db-master/pgbackrest.conf --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/db-master/lock +P00 INFO: sent term signal to process [PROCESS-ID] +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: stop command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +full backup - abort backup - local (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --type=full --stanza=db backup --test --test-delay=5 --test-point=backup-start=y +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --test --test-delay=5 --test-point=backup-start=y --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/backup/log +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = <0750>, strPathExp = /backup.history +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-1], [BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-1] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = false +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote, strCommandSSH = ssh, strHost = db-master, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no [USER-1]@db-master '[BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote', strId = 'db-master remote', strName = remote +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check(): bRequired = , iCatalogVersion = 201409291, iControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check=>: iDbHistoryId = 1 +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2], [BACKUP-FULL-1]) +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Protocol::Storage::Remote->exists(): strPathExp = [TEST_PATH]/db-master/db/base/postmaster.pid +P00 DEBUG: Protocol::Storage::Remote->exists=>: bExists = false +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = [undef], hDatabaseMap = [undef], hTablespaceMap = [undef], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = [undef], strParentPath = [undef], strPath = [TEST_PATH]/db-master/db/base +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base/pg_tblspc +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/pg_stat, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_stat +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/postgresql.conf, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_config/postgresql.conf +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_config/postgresql.conf +P00 WARN: aborted backup [BACKUP-FULL-2] of same type exists, will be cleaned to remove invalid files and resumed +P00 DEBUG: Backup::Backup->resumeClean(): oAbortedManifest = [object], oManifest = [object], oStorageRepo = [object], strBackupLabel = [BACKUP-FULL-2] +P00 DETAIL: clean resumed backup path: /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->manifest(): strPathExp = /[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Backup::Backup->processManifest(): bCompress = false, bHardLink = false, oBackupManifest = [object], strBackupLabel = [BACKUP-FULL-2], strDbCopyPath = [TEST_PATH]/db-master/db/base, strDbMasterPath = [TEST_PATH]/db-master/db/base, strDbVersion = 9.4, strLsnStart = [undef], strType = full +P00 TEST: PgBaCkReStTeSt-BACKUP-START-PgBaCkReStTeSt +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 ERROR: [063]: process 'db-master remote' terminated unexpectedly [063]: terminated on signal [SIGTERM] + +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = [undef], oException = [object], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = false, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: backup command end: terminated on signal from child process +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 63 + +full backup - global stop (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/backup/log +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = <0750>, strPathExp = /backup.history +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-1], [BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-1] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = false +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote, strCommandSSH = ssh, strHost = db-master, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no [USER-1]@db-master '[BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote', strId = 'db-master remote', strName = remote +P00 ERROR: [062]: process 'db-master remote' terminated unexpectedly [062]: stop file exists for all stanzas + +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = [undef], oException = [object], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = false, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: backup command end: aborted with exception [062] +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 62 + +stop db stanza (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db stop +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stop command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --stanza=db +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/db-master/lock +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: stop command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +stop db stanza (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db stop +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stop command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --stanza=db +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/db-master/lock +P00 WARN: stop file already exists for stanza db +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: stop command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +full backup - stanza stop (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/backup/log +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = <0750>, strPathExp = /backup.history +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-1], [BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-1] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = false +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote, strCommandSSH = ssh, strHost = db-master, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no [USER-1]@db-master '[BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote', strId = 'db-master remote', strName = remote +P00 ERROR: [062]: process 'db-master remote' terminated unexpectedly [062]: stop file exists for stanza db + +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = [undef], oException = [object], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = false, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: backup command end: aborted with exception [062] +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 62 + +start db stanza (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db start +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: start command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --stanza=db +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: start command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +start all stanzas (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf start +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: start command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: start command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +start all stanzas (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf start +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: start command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log +P00 WARN: stop file does not exist +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: start command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +stop all stanzas (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --force stop +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stop command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 INFO: sent term signal to process [PROCESS-ID] +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: stop command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +full backup - abort backup - remote (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --type=full --stanza=db backup --test --test-delay=5 --test-point=backup-start=y +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --test --test-delay=5 --test-point=backup-start=y --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/backup/log +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = <0750>, strPathExp = /backup.history +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-1], [BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-1] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = false +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote, strCommandSSH = ssh, strHost = db-master, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no [USER-1]@db-master '[BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote', strId = 'db-master remote', strName = remote +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check(): bRequired = , iCatalogVersion = 201409291, iControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check=>: iDbHistoryId = 1 +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2], [BACKUP-FULL-1]) +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Protocol::Storage::Remote->exists(): strPathExp = [TEST_PATH]/db-master/db/base/postmaster.pid +P00 DEBUG: Protocol::Storage::Remote->exists=>: bExists = false +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = [undef], hDatabaseMap = [undef], hTablespaceMap = [undef], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = [undef], strParentPath = [undef], strPath = [TEST_PATH]/db-master/db/base +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base/pg_tblspc +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/pg_stat, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_stat +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/postgresql.conf, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_config/postgresql.conf +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_config/postgresql.conf +P00 WARN: aborted backup [BACKUP-FULL-2] of same type exists, will be cleaned to remove invalid files and resumed +P00 DEBUG: Backup::Backup->resumeClean(): oAbortedManifest = [object], oManifest = [object], oStorageRepo = [object], strBackupLabel = [BACKUP-FULL-2] +P00 DETAIL: clean resumed backup path: /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->manifest(): strPathExp = /[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Backup::Backup->processManifest(): bCompress = false, bHardLink = false, oBackupManifest = [object], strBackupLabel = [BACKUP-FULL-2], strDbCopyPath = [TEST_PATH]/db-master/db/base, strDbMasterPath = [TEST_PATH]/db-master/db/base, strDbVersion = 9.4, strLsnStart = [undef], strType = full +P00 TEST: PgBaCkReStTeSt-BACKUP-START-PgBaCkReStTeSt +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 63, oException = [undef], strSignal = TERM +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = false, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = db +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 ERROR: [063]: terminated on signal [SIGTERM] +P00 INFO: backup command end: terminated on signal [SIGTERM] +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 63 + +full backup - global stop (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 ERROR: [062]: stop file exists for all stanzas +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = [undef], oException = [object], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = false, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: backup command end: aborted with exception [062] +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 62 + +start all stanzas (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf start +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: start command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: start command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +full backup - resume (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --force --checksum-page --type=full --stanza=db backup --test --test-delay=0.2 --test-point=backup-resume=y +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --checksum-page --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --test --test-delay=0.2 --test-point=backup-resume=y --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/backup/log +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = <0750>, strPathExp = /backup.history +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = false +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-1] +P00 DEBUG: Storage::Local->pathExists=>: bExists = false +P00 WARN: backup [BACKUP-FULL-1] missing in repository removed from backup.info +P00 DEBUG: Backup::Info->delete(): strBackupLabel = [BACKUP-FULL-1] +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote, strCommandSSH = ssh, strHost = db-master, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no [USER-1]@db-master '[BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote', strId = 'db-master remote', strName = remote +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check(): bRequired = , iCatalogVersion = 201409291, iControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check=>: iDbHistoryId = 1 +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2]) +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Protocol::Storage::Remote->exists(): strPathExp = [TEST_PATH]/db-master/db/base/postmaster.pid +P00 DEBUG: Protocol::Storage::Remote->exists=>: bExists = true +P00 WARN: --no-online passed and postmaster.pid exists but --force was passed so backup will continue though it looks like the postmaster is running and the backup will probably not be consistent +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = [undef], hDatabaseMap = [undef], hTablespaceMap = [undef], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = [undef], strParentPath = [undef], strPath = [TEST_PATH]/db-master/db/base +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base/pg_tblspc +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/pg_stat, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_stat +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = false, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = pg_data/postgresql.conf, strParentPath = [TEST_PATH]/db-master/db/base, strPath = ../pg_config/postgresql.conf +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_config/postgresql.conf +P00 WARN: aborted backup [BACKUP-FULL-2] of same type exists, will be cleaned to remove invalid files and resumed +P00 TEST: PgBaCkReStTeSt-BACKUP-RESUME-PgBaCkReStTeSt +P00 DEBUG: Backup::Backup->resumeClean(): oAbortedManifest = [object], oManifest = [object], oStorageRepo = [object], strBackupLabel = [BACKUP-FULL-2] +P00 DETAIL: clean resumed backup path: /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->manifest(): strPathExp = /[BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->resumeClean: remove file file.tmp +P00 DEBUG: Backup::Backup->resumeClean: remove file pg_data/PG_VERSION +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = (/[BACKUP-FULL-2]/file.tmp, /[BACKUP-FULL-2]/pg_data/PG_VERSION) +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Backup::Backup->processManifest(): bCompress = false, bHardLink = false, oBackupManifest = [object], strBackupLabel = [BACKUP-FULL-2], strDbCopyPath = [TEST_PATH]/db-master/db/base, strDbMasterPath = [TEST_PATH]/db-master/db/base, strDbVersion = 9.4, strLsnStart = [undef], strType = full +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Local::Process->new(): bConfessError = , iSelectTimeout = <30>, strBackRestBin = <[BACKREST-BIN]>, strHostType = db +P00 DEBUG: Protocol::Local::Process->hostAdd(): iHostConfigIdx = 1, iProcessMax = 1 +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-2]/pg_data +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-2]/pg_data/base +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-2]/pg_data/base/1 +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-2]/pg_data/base/16384 +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-2]/pg_data/base/32768 +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-2]/pg_data/global +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-2]/pg_data/pg_clog +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-2]/pg_data/pg_stat +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = true, strMode = <0750>, strPathExp = /[BACKUP-FULL-2]/pg_data/pg_tblspc +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/33001, pg_data/base/32768/33001, 65536, 6bf316f11d28c28914ea9be92c00de9bea6d9a6b, 1, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/32768/33001, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/33000.32767, pg_data/base/32768/33000.32767, 32768, 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5, 1, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/32768/33000.32767, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/33000, pg_data/base/32768/33000, 32768, 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f, 1, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/32768/33000, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/16384/17000, pg_data/base/16384/17000, 16384, e0101dd8ffb910c9c202ca35b5f828bcb9697bed, 1, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/16384/17000, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/global/pg_control, pg_data/global/pg_control, 8192, 89373d9f2973502940de06bc5212489df3f8a912, 0, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-2], 0, [undef]), strKey = pg_data/global/pg_control, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/1/12000, pg_data/base/1/12000, 8192, 22c98d248ff548311eda88559e4a8405ed77c003, 1, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_data/base/1/12000, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/postgresql.conf, pg_data/postgresql.conf, 21, 6721d92c9fcdf4248acff1f9a1377127d9064807, 0, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-2], 1, [undef]), strKey = pg_data/postgresql.conf, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/pg_stat/global.stat, pg_data/pg_stat/global.stat, 5, e350d5ce0153f3e22d5db21cf2a4eff00f3ee877, 0, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-2], 1, [undef]), strKey = pg_data/pg_stat/global.stat, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/PG_VERSION, pg_data/base/32768/PG_VERSION, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, 0, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/base/32768/PG_VERSION, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/16384/PG_VERSION, pg_data/base/16384/PG_VERSION, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, 0, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/base/16384/PG_VERSION, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/1/PG_VERSION, pg_data/base/1/PG_VERSION, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, 0, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/base/1/PG_VERSION, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/PG_VERSION, pg_data/PG_VERSION, 3, [undef], 0, [BACKUP-FULL-2], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/PG_VERSION, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->hostConnect: start local process: iHostConfigIdx = 1, iHostIdx = 0, iHostProcessIdx = 0, iProcessId = 1, strHostType = db +P00 DEBUG: Protocol::Local::Master->new(): iProcessIdx = 1, strCommand = [BACKREST-BIN] --command=backup --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --host-id=1 --lock-path=[TEST_PATH]/backup/lock --log-path=[TEST_PATH]/backup/log --process=1 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db local +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --command=backup --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --host-id=1 --lock-path=[TEST_PATH]/backup/lock --log-path=[TEST_PATH]/backup/log --process=1 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db local, strId = 'local-1', strName = local +P00 DEBUG: Protocol::Local::Process->hostConnect=>: bResult = true +P00 DEBUG: Protocol::Local::Process->init: init local process: iDirection = 1, iHostIdx = 0, iProcessId = 1, iQueueIdx = 0, iQueueLastIdx = 0 +P00 DEBUG: Protocol::Local::Process->init=>: bResult = true +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/33001, strQueueIdx = 0 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 65536, 65536, 6bf316f11d28c28914ea9be92c00de9bea6d9a6b, [undef]), strKey = pg_data/base/32768/33001 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/33000.32767, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/32768/33001 (64KB, 39%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 32768, 32768, 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5, [undef]), strKey = pg_data/base/32768/33000.32767 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/33000, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/32768/33000.32767 (32KB, 59%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 32768, 32768, 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f, [undef]), strKey = pg_data/base/32768/33000 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/16384/17000, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/32768/33000 (32KB, 79%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 16384, 16384, e0101dd8ffb910c9c202ca35b5f828bcb9697bed, [undef]), strKey = pg_data/base/16384/17000 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/global/pg_control, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/16384/17000 (16KB, 89%) checksum e0101dd8ffb910c9c202ca35b5f828bcb9697bed +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 8192, 8192, 89373d9f2973502940de06bc5212489df3f8a912, [undef]), strKey = pg_data/global/pg_control +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/1/12000, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/global/pg_control (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 8192, 8192, 22c98d248ff548311eda88559e4a8405ed77c003, [undef]), strKey = pg_data/base/1/12000 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/postgresql.conf, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/1/12000 (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 21, 21, 6721d92c9fcdf4248acff1f9a1377127d9064807, [undef]), strKey = pg_data/postgresql.conf +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/pg_stat/global.stat, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/postgresql.conf (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 5, 5, e350d5ce0153f3e22d5db21cf2a4eff00f3ee877, [undef]), strKey = pg_data/pg_stat/global.stat +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/PG_VERSION, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/pg_stat/global.stat (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 3, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, [undef]), strKey = pg_data/base/32768/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/16384/PG_VERSION, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/32768/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 3, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, [undef]), strKey = pg_data/base/16384/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/1/PG_VERSION, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/16384/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 3, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, [undef]), strKey = pg_data/base/1/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/PG_VERSION, strQueueIdx = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/1/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 3, 3, 184473f470864e067ee3a22e64b47b0a1c356f29, [undef]), strKey = pg_data/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: no jobs found, stop local: iHostConfigIdx = [undef], iHostIdx = 0, iProcessId = 1, strHostType = [undef] +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/PG_VERSION (3B, 100%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Protocol::Local::Process->process: all jobs complete +P00 DEBUG: Backup::Backup->processManifest=>: lSizeTotal = 163878 +P00 INFO: full backup size = 160KB +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = db +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Storage::Local->pathSync(): bRecurse = true, strPathExp = /[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db/[BACKUP-FULL-2] +P00 INFO: new backup label = [BACKUP-FULL-2] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = true, bPathCreate = true, lTimestamp = [undef], rhyFilter = ({strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup.history/[YEAR-1]/[BACKUP-FULL-2].manifest.gz +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = /[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup.history +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = /latest +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/backup.info.copy +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: expire command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-FULL-2]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-FULL-2]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="full" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base","type":"path"} +pg_data/pg_stat={"path":"../pg_stat","type":"link"} +pg_data/postgresql.conf={"file":"postgresql.conf","path":"../pg_config","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"e0101dd8ffb910c9c202ca35b5f828bcb9697bed","checksum-page":false,"checksum-page-error":[1],"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-2]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_stat={"destination":"../pg_stat"} +pg_data/postgresql.conf={"destination":"../pg_config/postgresql.conf"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +full backup - invalid repo (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --repo-path=/bogus_path --log-level-console=detail --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/bogus_path --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 ERROR: [055]: unable to open /bogus_path/backup/db/backup.info or /bogus_path/backup/db/backup.info.copy +P00 ERROR: [055]: /bogus_path/backup/db/backup.info does not exist and is required to perform a backup. + HINT: has a stanza-create been performed? +P00 INFO: backup command end: aborted with exception [055] + +restore delta, backup '[BACKUP-FULL-2]' - add and delete files (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-FULL-2] --link-all --cmd-ssh=/usr/bin/ssh --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --cmd-ssh=/usr/bin/ssh --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --delta --link-all --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --set=[BACKUP-FULL-2] --stanza=db +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = restore +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/db-master/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/db-master/log +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=restore --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = /usr/bin/ssh, strHost = backup, strUser = [USER-2] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = /usr/bin/ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=restore --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = [TEST_PATH]/db-master/db/base/postmaster.pid +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->exists(): strFileExp = [TEST_PATH]/db-master/db/base/PG_VERSION +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /backup.info +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [TEST_PATH]/db-master/db/base/backup.info, xSourceFile = [object] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/backup.info +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = false, oStorage = [object], strBackupClusterPath = [TEST_PATH]/db-master/db/base +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/db/base/backup.info +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = [TEST_PATH]/db-master/db/base/backup.info +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 INFO: restore backup set [BACKUP-FULL-2] +P00 DEBUG: Protocol::Storage::Remote->exists(): strPathExp = /[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Protocol::Storage::Remote->exists=>: bExists = true +P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [TEST_PATH]/db-master/db/base/backup.manifest, xSourceFile = [object] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/backup.manifest +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/db/base/backup.manifest +P00 DEBUG: Restore->manifestOwnershipCheck(): oManifest = [object] +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = [TEST_PATH]/db-master/db/base/global/pg_control +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DEBUG: Restore->clean(): oManifest = [object] +P00 DETAIL: check [TEST_PATH]/db-master/db/base exists +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->manifest(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DETAIL: check [TEST_PATH]/db-master/db/pg_stat exists +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 DETAIL: check [TEST_PATH]/db-master/db/pg_config exists +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/pg_config +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = [TEST_PATH]/db-master/db/pg_config/postgresql.conf +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/pg_config +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/pg_config +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_config +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/pg_stat +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->manifest(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/base +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->manifest(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DETAIL: remove file [TEST_PATH]/db-master/db/base/recovery.done +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = [TEST_PATH]/db-master/db/base/recovery.done +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DETAIL: preserve file [TEST_PATH]/db-master/db/base/recovery.conf +P00 DETAIL: remove file [TEST_PATH]/db-master/db/base/deleteme/deleteme.txt +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = [TEST_PATH]/db-master/db/base/deleteme/deleteme.txt +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DETAIL: remove path [TEST_PATH]/db-master/db/base/deleteme +P00 DETAIL: set mode 0700 on [TEST_PATH]/db-master/db/base/base +P00 DETAIL: remove file [TEST_PATH]/db-master/db/base/backup_label.old +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = [TEST_PATH]/db-master/db/base/backup_label.old +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 INFO: cleanup removed 3 files, 1 path +P00 DEBUG: Restore->build(): oManifest = [object] +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/pg_stat +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/pg_config +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: build level 1 paths/links +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base/base +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base/global +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base/pg_clog +P00 DEBUG: Storage::Local->pathExists=>: bExists = false +P00 DEBUG: Storage::Local->exists(): strFileExp = [TEST_PATH]/db-master/db/base/pg_clog +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = , strMode = 0700, strPathExp = [TEST_PATH]/db-master/db/base/pg_clog +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base/pg_tblspc +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base/pg_stat +P00 DEBUG: Storage::Local->pathExists=>: bExists = false +P00 DEBUG: Storage::Local->exists(): strFileExp = [TEST_PATH]/db-master/db/base/pg_stat +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base/postgresql.conf +P00 DEBUG: Storage::Local->pathExists=>: bExists = false +P00 DEBUG: Storage::Local->exists(): strFileExp = [TEST_PATH]/db-master/db/base/postgresql.conf +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: build level 2 paths/links +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base/base/1 +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base/base/16384 +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = [TEST_PATH]/db-master/db/base/base/32768 +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: build level 3 paths/links +P00 DEBUG: Protocol::Local::Process->new(): bConfessError = , iSelectTimeout = <30>, strBackRestBin = <[BACKREST-BIN]>, strHostType = backup +P00 DEBUG: Protocol::Local::Process->hostAdd(): iHostConfigIdx = 1, iProcessMax = 1 +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/33001, 65536, [MODIFICATION-TIME-1], 6bf316f11d28c28914ea9be92c00de9bea6d9a6b, 0, 0, pg_data/base/32768/33001, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/base/32768/33001, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/33000.32767, 32768, [MODIFICATION-TIME-1], 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5, 0, 0, pg_data/base/32768/33000.32767, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/base/32768/33000.32767, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/33000, 32768, [MODIFICATION-TIME-1], 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f, 0, 0, pg_data/base/32768/33000, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/base/32768/33000, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/16384/17000, 16384, [MODIFICATION-TIME-1], e0101dd8ffb910c9c202ca35b5f828bcb9697bed, 0, 0, pg_data/base/16384/17000, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/base/16384/17000, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/global/pg_control.pgbackrest.tmp, 8192, [MODIFICATION-TIME-2], 89373d9f2973502940de06bc5212489df3f8a912, 0, 0, pg_data/global/pg_control, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/global/pg_control, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/1/12000, 8192, [MODIFICATION-TIME-1], 22c98d248ff548311eda88559e4a8405ed77c003, 0, 0, pg_data/base/1/12000, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/base/1/12000, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/postgresql.conf, 21, [MODIFICATION-TIME-2], 6721d92c9fcdf4248acff1f9a1377127d9064807, 0, 0, pg_data/postgresql.conf, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/postgresql.conf, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/pg_stat/global.stat, 5, [MODIFICATION-TIME-2], e350d5ce0153f3e22d5db21cf2a4eff00f3ee877, 0, 0, pg_data/pg_stat/global.stat, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/pg_stat/global.stat, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/32768/PG_VERSION, 3, [MODIFICATION-TIME-1], 184473f470864e067ee3a22e64b47b0a1c356f29, 0, 0, pg_data/base/32768/PG_VERSION, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/base/32768/PG_VERSION, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/16384/PG_VERSION, 3, [MODIFICATION-TIME-1], 184473f470864e067ee3a22e64b47b0a1c356f29, 0, 0, pg_data/base/16384/PG_VERSION, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/base/16384/PG_VERSION, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/base/1/PG_VERSION, 3, [MODIFICATION-TIME-1], 184473f470864e067ee3a22e64b47b0a1c356f29, 0, 0, pg_data/base/1/PG_VERSION, [undef], 0660, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/base/1/PG_VERSION, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/PG_VERSION, 3, [MODIFICATION-TIME-1], 184473f470864e067ee3a22e64b47b0a1c356f29, 0, 0, pg_data/PG_VERSION, [undef], 0600, [USER-1], [GROUP-1], [MODIFICATION-TIME-3], 1, [BACKUP-FULL-2], 0), strKey = pg_data/PG_VERSION, strOp = restoreFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->hostConnect: start local process: iHostConfigIdx = 1, iHostIdx = 0, iHostProcessIdx = 0, iProcessId = 1, strHostType = backup +P00 DEBUG: Protocol::Local::Master->new(): iProcessIdx = 1, strCommand = [BACKREST-BIN] --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --cmd-ssh=/usr/bin/ssh --command=restore --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --host-id=1 --lock-path=[TEST_PATH]/db-master/lock --log-path=[TEST_PATH]/db-master/log --process=1 --protocol-timeout=60 --stanza=db --type=backup local +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --cmd-ssh=/usr/bin/ssh --command=restore --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --host-id=1 --lock-path=[TEST_PATH]/db-master/lock --log-path=[TEST_PATH]/db-master/log --process=1 --protocol-timeout=60 --stanza=db --type=backup local, strId = 'local-1', strName = local +P00 DEBUG: Protocol::Local::Process->hostConnect=>: bResult = true +P00 DEBUG: Protocol::Local::Process->init: init local process: iDirection = 1, iHostIdx = 0, iProcessId = 1, iQueueIdx = 0, iQueueLastIdx = 0 +P00 DEBUG: Protocol::Local::Process->init=>: bResult = true +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/33001, strQueueIdx = 0 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/base/32768/33001 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/33000.32767, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-1], lSize = 65536, lSizeCurrent = 0, lSizeTotal = 163878, strChecksum = 6bf316f11d28c28914ea9be92c00de9bea6d9a6b, strDbFile = [TEST_PATH]/db-master/db/base/base/32768/33001 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/33001 - exists and matches backup (64KB, 39%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/base/32768/33000.32767 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/33000, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-1], lSize = 32768, lSizeCurrent = 65536, lSizeTotal = 163878, strChecksum = 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5, strDbFile = [TEST_PATH]/db-master/db/base/base/32768/33000.32767 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/33000.32767 - exists and matches backup (32KB, 59%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/base/32768/33000 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/16384/17000, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-1], lSize = 32768, lSizeCurrent = 98304, lSizeTotal = 163878, strChecksum = 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f, strDbFile = [TEST_PATH]/db-master/db/base/base/32768/33000 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/33000 - exists and matches backup (32KB, 79%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1), strKey = pg_data/base/16384/17000 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/global/pg_control, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = true, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-1], lSize = 16384, lSizeCurrent = 131072, lSizeTotal = 163878, strChecksum = e0101dd8ffb910c9c202ca35b5f828bcb9697bed, strDbFile = [TEST_PATH]/db-master/db/base/base/16384/17000 +P01 INFO: restore file [TEST_PATH]/db-master/db/base/base/16384/17000 (16KB, 89%) checksum e0101dd8ffb910c9c202ca35b5f828bcb9697bed +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1), strKey = pg_data/global/pg_control +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/1/12000, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = true, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-2], lSize = 8192, lSizeCurrent = 147456, lSizeTotal = 163878, strChecksum = 89373d9f2973502940de06bc5212489df3f8a912, strDbFile = [TEST_PATH]/db-master/db/base/global/pg_control.pgbackrest.tmp +P01 INFO: restore file [TEST_PATH]/db-master/db/base/global/pg_control.pgbackrest.tmp (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/base/1/12000 +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/postgresql.conf, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-1], lSize = 8192, lSizeCurrent = 155648, lSizeTotal = 163878, strChecksum = 22c98d248ff548311eda88559e4a8405ed77c003, strDbFile = [TEST_PATH]/db-master/db/base/base/1/12000 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/1/12000 - exists and matches backup (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/postgresql.conf +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/pg_stat/global.stat, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-2], lSize = 21, lSizeCurrent = 163840, lSizeTotal = 163878, strChecksum = 6721d92c9fcdf4248acff1f9a1377127d9064807, strDbFile = [TEST_PATH]/db-master/db/base/postgresql.conf +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/postgresql.conf - exists and matches backup (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/pg_stat/global.stat +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/32768/PG_VERSION, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-2], lSize = 5, lSizeCurrent = 163861, lSizeTotal = 163878, strChecksum = e350d5ce0153f3e22d5db21cf2a4eff00f3ee877, strDbFile = [TEST_PATH]/db-master/db/base/pg_stat/global.stat +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/pg_stat/global.stat - exists and matches backup (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/base/32768/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/16384/PG_VERSION, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-1], lSize = 3, lSizeCurrent = 163866, lSizeTotal = 163878, strChecksum = 184473f470864e067ee3a22e64b47b0a1c356f29, strDbFile = [TEST_PATH]/db-master/db/base/base/32768/PG_VERSION +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/base/16384/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/base/1/PG_VERSION, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-1], lSize = 3, lSizeCurrent = 163869, lSizeTotal = 163878, strChecksum = 184473f470864e067ee3a22e64b47b0a1c356f29, strDbFile = [TEST_PATH]/db-master/db/base/base/16384/PG_VERSION +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/16384/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/base/1/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/PG_VERSION, strQueueIdx = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-1], lSize = 3, lSizeCurrent = 163872, lSizeTotal = 163878, strChecksum = 184473f470864e067ee3a22e64b47b0a1c356f29, strDbFile = [TEST_PATH]/db-master/db/base/base/1/PG_VERSION +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/1/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0), strKey = pg_data/PG_VERSION +P00 DEBUG: Protocol::Local::Process->process: no jobs found, stop local: iHostConfigIdx = [undef], iHostIdx = 0, iProcessId = 1, strHostType = [undef] +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: RestoreFile::restoreLog(): bCopy = false, bForce = false, bZero = false, iLocalId = 1, lModificationTime = [MODIFICATION-TIME-1], lSize = 3, lSizeCurrent = 163875, lSizeTotal = 163878, strChecksum = 184473f470864e067ee3a22e64b47b0a1c356f29, strDbFile = [TEST_PATH]/db-master/db/base/PG_VERSION +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/PG_VERSION - exists and matches backup (3B, 100%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 DEBUG: Protocol::Local::Process->process: all jobs complete +P00 DEBUG: Storage::Local->exists(): strFileExp = [TEST_PATH]/db-master/db/base/recovery.conf +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = [TEST_PATH]/db-master/db/base/recovery.conf +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 INFO: write [TEST_PATH]/db-master/db/base/recovery.conf +P00 DEBUG: Storage::Local->pathSync(): bRecurse = true, strPathExp = [TEST_PATH]/db-master/db/base +P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started) +P00 DEBUG: Storage::Local->move(): bPathCreate = , strDestinationPathExp = [TEST_PATH]/db-master/db/base/global/pg_control, strSourcePathExp = [TEST_PATH]/db-master/db/base/global/pg_control.pgbackrest.tmp +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = [TEST_PATH]/db-master/db/base/backup.manifest +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: restore command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --cmd-ssh=/usr/bin/ssh --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"' + +restore delta, backup '[BACKUP-FULL-2]' - fix broken symlink (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-FULL-2] --link-all --log-level-console=detail --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --delta --link-all --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --set=[BACKUP-FULL-2] --stanza=db +P00 INFO: restore backup set [BACKUP-FULL-2] +P00 DETAIL: check [TEST_PATH]/db-master/db/base exists +P00 DETAIL: check [TEST_PATH]/db-master/db/pg_stat exists +P00 DETAIL: check [TEST_PATH]/db-master/db/pg_config exists +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/pg_config +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/pg_stat +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/base +P00 DETAIL: preserve file [TEST_PATH]/db-master/db/base/recovery.conf +P00 DETAIL: remove link [TEST_PATH]/db-master/db/base/pg_stat - destination changed +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/33001 - exists and matches backup (64KB, 39%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/33000.32767 - exists and matches backup (32KB, 59%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/33000 - exists and matches backup (32KB, 79%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/16384/17000 - exists and matches backup (16KB, 89%) checksum e0101dd8ffb910c9c202ca35b5f828bcb9697bed +P01 INFO: restore file [TEST_PATH]/db-master/db/base/global/pg_control.pgbackrest.tmp (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/1/12000 - exists and matches backup (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/postgresql.conf - exists and matches backup (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/pg_stat/global.stat - exists and matches backup (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/16384/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/1/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/PG_VERSION - exists and matches backup (3B, 100%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 INFO: write [TEST_PATH]/db-master/db/base/recovery.conf +P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started) +P00 INFO: restore command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-get %f "%p"' + +restore delta, force, backup '[BACKUP-FULL-2]', expect exit 40 - fail on missing PG_VERSION (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --force --set=[BACKUP-FULL-2] --log-level-console=detail --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --delta --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --set=[BACKUP-FULL-2] --stanza=db +P00 WARN: --delta or --force specified but unable to find 'PG_VERSION' or 'backup.manifest' in '[TEST_PATH]/db-master/db/base' to confirm that this is a valid $PGDATA directory. --delta and --force have been disabled and if any files exist in the destination directories the restore will be aborted. +P00 INFO: restore backup set [BACKUP-FULL-2] +P00 WARN: contents of directory link pg_stat will be restored in a directory at the same location +P00 WARN: file link postgresql.conf will be restored as a file at the same location +P00 DETAIL: check [TEST_PATH]/db-master/db/base exists +P00 ERROR: [040]: cannot restore to path '[TEST_PATH]/db-master/db/base' that contains files - try using --delta if this is what you intended +P00 INFO: restore command end: aborted with exception [040] + +restore delta, force, backup '[BACKUP-FULL-2]' - restore succeeds with backup.manifest file (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --force --set=[BACKUP-FULL-2] --log-level-console=detail --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --delta --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --set=[BACKUP-FULL-2] --stanza=db +P00 INFO: restore backup set [BACKUP-FULL-2] +P00 WARN: group bogus in manifest cannot be used for restore, set to [USER-1] +P00 WARN: user bogus in manifest cannot be used for restore, set to [USER-1] +P00 WARN: contents of directory link pg_stat will be restored in a directory at the same location +P00 WARN: file link postgresql.conf will be restored as a file at the same location +P00 DETAIL: check [TEST_PATH]/db-master/db/base exists +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/base +P00 DETAIL: preserve file [TEST_PATH]/db-master/db/base/recovery.conf +P00 DETAIL: remove link [TEST_PATH]/db-master/db/base/postgresql.conf +P00 DETAIL: remove link [TEST_PATH]/db-master/db/base/pg_stat +P00 INFO: cleanup removed 2 links +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/33001 - exists and matches size 65536 and modification time [MODIFICATION-TIME-1] (64KB, 39%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/33000.32767 - exists and matches size 32768 and modification time [MODIFICATION-TIME-1] (32KB, 59%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/33000 - exists and matches size 32768 and modification time [MODIFICATION-TIME-1] (32KB, 79%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/16384/17000 - exists and matches size 16384 and modification time [MODIFICATION-TIME-1] (16KB, 89%) checksum e0101dd8ffb910c9c202ca35b5f828bcb9697bed +P01 INFO: restore file [TEST_PATH]/db-master/db/base/global/pg_control.pgbackrest.tmp (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/1/12000 - exists and matches size 8192 and modification time [MODIFICATION-TIME-1] (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P01 INFO: restore file [TEST_PATH]/db-master/db/base/postgresql.conf (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P01 INFO: restore file [TEST_PATH]/db-master/db/base/pg_stat/global.stat (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/32768/PG_VERSION - exists and matches size 3 and modification time [MODIFICATION-TIME-1] (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/16384/PG_VERSION - exists and matches size 3 and modification time [MODIFICATION-TIME-1] (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base/base/1/PG_VERSION - exists and matches size 3 and modification time [MODIFICATION-TIME-1] (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base/PG_VERSION (3B, 100%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P00 INFO: write [TEST_PATH]/db-master/db/base/recovery.conf +P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started) +P00 INFO: restore command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf +---------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-get %f "%p"' + +incr backup - invalid database version (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 ERROR: [051]: database version = 9.4, system-id 6353949018581704918 does not match backup version = 8.0, system-id = 6353949018581704918 + HINT: is this the correct stanza? +P00 INFO: backup command end: aborted with exception [051] + +incr backup - invalid system id (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 ERROR: [051]: database version = 9.4, system-id 6353949018581704918 does not match backup version = 9.4, system-id = 6999999999999999999 + HINT: is this the correct stanza? +P00 INFO: backup command end: aborted with exception [051] + +incr backup - invalid control version (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 ERROR: [051]: database control-version = 942, catalog-version 201409291 does not match backup control-version = 842, catalog-version = 201409291 + HINT: this may be a symptom of database or repository corruption! +P00 INFO: backup command end: aborted with exception [051] + +incr backup - invalid catalog version (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 ERROR: [051]: database control-version = 942, catalog-version 201409291 does not match backup control-version = 942, catalog-version = 197208141 + HINT: this may be a symptom of database or repository corruption! +P00 INFO: backup command end: aborted with exception [051] + +incr backup - invalid path in pg_tblspc (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] +P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] +P00 ERROR: [069]: pg_tblspc/path is not a symlink - pg_tblspc should contain only symlinks +P00 INFO: backup command end: aborted with exception [069] + +incr backup - invalid relative tablespace in $PGDATA (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] +P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] +P00 ERROR: [071]: tablespace symlink ../invalid_tblspc destination must not be in $PGDATA +P00 INFO: backup command end: aborted with exception [071] + +incr backup - invalid tablespace in $PGDATA (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] +P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] +P00 ERROR: [071]: tablespace symlink [TEST_PATH]/db-master/db/base/invalid_tblspc destination must not be in $PGDATA +P00 INFO: backup command end: aborted with exception [071] + +incr backup - add tablespace 1 (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --test --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --test +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/backup/log +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = <0750>, strPathExp = /backup.history +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote, strCommandSSH = ssh, strHost = db-master, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no [USER-1]@db-master '[BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote', strId = 'db-master remote', strName = remote +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check(): bRequired = , iCatalogVersion = 201409291, iControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check=>: iDbHistoryId = 1 +P00 DEBUG: Backup::Info->last(): strType = incr +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Backup::Info->list(): strFilter = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strOrder = reverse +P00 DEBUG: Backup::Info->list=>: stryBackup = ([BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->last=>: strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->dbHistoryList=>: hDbHash = [hash] +P00 DEBUG: Backup::Info->confirmDb=>: bConfirmDb = true +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2]) +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-2]/backup.manifest.copy +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = [BACKUP-EXPR](D|I)$, strPathExp = , strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = () +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = [BACKUP-EXPR](D|I)\.manifest\.gz$, strPathExp = /backup.history/[YEAR-1], strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = () +P00 DEBUG: Protocol::Storage::Remote->exists(): strPathExp = [TEST_PATH]/db-master/db/base/postmaster.pid +P00 DEBUG: Protocol::Storage::Remote->exists=>: bExists = false +P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = [undef], hDatabaseMap = [undef], hTablespaceMap = [undef], oLastManifest = [object], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = [undef], strParentPath = [undef], strPath = [TEST_PATH]/db-master/db/base +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base/pg_tblspc +P00 DEBUG: Manifest->build: found tablespace 1 in offline mode +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = true, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [TS_PATH-1], strLevel = pg_tblspc/1, strParentPath = [TEST_PATH]/db-master/db/base/pg_tblspc, strPath = [TEST_PATH]/db-master/db/tablespace/ts1 +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/tablespace/ts1 +P00 DEBUG: Backup::Backup->process: create backup path /backup/db/[BACKUP-INCR-1] +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = , bIgnoreExists = , strMode = <0750>, strPathExp = /[BACKUP-INCR-1] +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-INCR-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-INCR-1]/backup.manifest.copy +P00 DEBUG: Backup::Backup->processManifest(): bCompress = false, bHardLink = false, oBackupManifest = [object], strBackupLabel = [BACKUP-INCR-1], strDbCopyPath = [TEST_PATH]/db-master/db/base, strDbMasterPath = [TEST_PATH]/db-master/db/base, strDbVersion = 9.4, strLsnStart = [undef], strType = incr +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Local::Process->new(): bConfessError = , iSelectTimeout = <30>, strBackRestBin = <[BACKREST-BIN]>, strHostType = db +P00 DEBUG: Protocol::Local::Process->hostAdd(): iHostConfigIdx = 1, iProcessMax = 1 +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/32768/33001 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/32768/33000 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/16384/17000 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/global/pg_control to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/1/12000 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/postgresql.conf to [BACKUP-FULL-2] +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/badchecksum.txt, pg_data/badchecksum.txt, 11, [undef], 0, [BACKUP-INCR-1], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/badchecksum.txt, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt, pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt, 7, [undef], 1, [BACKUP-INCR-1], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt, strOp = backupFile, strQueue = pg_tblspc/1 +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/PG_VERSION to [BACKUP-FULL-2] +P00 DEBUG: Protocol::Local::Process->hostConnect: start local process: iHostConfigIdx = 1, iHostIdx = 0, iHostProcessIdx = 0, iProcessId = 1, strHostType = db +P00 DEBUG: Protocol::Local::Master->new(): iProcessIdx = 1, strCommand = [BACKREST-BIN] --command=backup --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --host-id=1 --lock-path=[TEST_PATH]/backup/lock --log-path=[TEST_PATH]/backup/log --process=1 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db local +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --command=backup --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --host-id=1 --lock-path=[TEST_PATH]/backup/lock --log-path=[TEST_PATH]/backup/log --process=1 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db local, strId = 'local-1', strName = local +P00 DEBUG: Protocol::Local::Process->hostConnect=>: bResult = true +P00 DEBUG: Protocol::Local::Process->init: init local process: iDirection = 1, iHostIdx = 0, iProcessId = 1, iQueueIdx = 0, iQueueLastIdx = 1 +P00 DEBUG: Protocol::Local::Process->init=>: bResult = true +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/badchecksum.txt, strQueueIdx = 0 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 11, 11, f927212cd08d11a42a666b2f04235398e9ceeb51, [undef]), strKey = pg_data/badchecksum.txt +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt, strQueueIdx = 1 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/badchecksum.txt (11B, 61%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 7, 7, d85de07d6421d90aa9191c11c889bfde43680f0f, {bAlign => 0, bValid => 0}), strKey = pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt +P00 DEBUG: Protocol::Local::Process->process: no jobs found, stop local: iHostConfigIdx = [undef], iHostIdx = 0, iProcessId = 1, strHostType = [undef] +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 100%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192 +P00 DEBUG: Protocol::Local::Process->process: all jobs complete +P00 DEBUG: Backup::Backup->processManifest=>: lSizeTotal = 18 +P00 INFO: incr backup size = 18B +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = db +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Storage::Local->pathSync(): bRecurse = true, strPathExp = /[BACKUP-INCR-1] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-INCR-1]/backup.manifest +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db/[BACKUP-INCR-1] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-INCR-1]/backup.manifest.copy +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db/[BACKUP-INCR-1] +P00 INFO: new backup label = [BACKUP-INCR-1] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = true, bPathCreate = true, lTimestamp = [undef], rhyFilter = ({strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup.history/[YEAR-1]/[BACKUP-INCR-1].manifest.gz +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = /[BACKUP-INCR-1]/backup.manifest +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-INCR-1]/backup.manifest +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup.history +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = /latest +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/backup.info.copy +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2], [BACKUP-INCR-1]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-INCR-1] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-INCR-1] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-INCR-1]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-INCR-1], [BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-INCR-1] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: expire command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-INCR-1]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-INCR-1]" +backup-prior="[BACKUP-FULL-2]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="incr" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base","type":"path"} +pg_tblspc/1={"path":"[TEST_PATH]/db-master/db/tablespace/ts1","tablespace-id":"1","tablespace-name":"ts1","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"e0101dd8ffb910c9c202ca35b5f828bcb9697bed","checksum-page":false,"checksum-page-error":[1],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt={"checksum":"d85de07d6421d90aa9191c11c889bfde43680f0f","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/1={"destination":"[TEST_PATH]/db-master/db/tablespace/ts1"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/1={} +pg_tblspc/1/[TS_PATH-1]={} +pg_tblspc/1/[TS_PATH-1]/16384={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-1]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +incr backup - resume and add tablespace 2 (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --stanza=db backup --test --test-delay=0.2 --test-point=backup-resume=y +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --test --test-delay=0.2 --test-point=backup-resume=y +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 DEBUG: Common::Lock::lockAcquire(): bFailOnNoLock = , bRemote = , strLockType = backup +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 770, strPathExp = [TEST_PATH]/backup/lock +P00 DEBUG: Common::Lock::lockAcquire=>: bResult = true +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = 0770, strPathExp = [TEST_PATH]/backup/log +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->pathCreate(): bCreateParent = true, bIgnoreExists = true, strMode = <0750>, strPathExp = /backup.history +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2], [BACKUP-INCR-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-INCR-2] +P00 DEBUG: Backup::Info->current=>: bTest = false +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-INCR-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-INCR-1] +P00 DEBUG: Storage::Local->pathExists=>: bExists = false +P00 WARN: backup [BACKUP-INCR-1] missing in repository removed from backup.info +P00 DEBUG: Backup::Info->delete(): strBackupLabel = [BACKUP-INCR-1] +P00 DEBUG: Db->new(): iRemoteIdx = 1 +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: create (cached) remote protocol +P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote, strCommandSSH = ssh, strHost = db-master, strUser = [USER-1] +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no [USER-1]@db-master '[BACKREST-BIN] --buffer-size=4194304 --command=backup --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db remote', strId = 'db-master remote', strName = remote +P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object] +P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> +P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check(): bRequired = , iCatalogVersion = 201409291, iControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Backup::Info->check=>: iDbHistoryId = 1 +P00 DEBUG: Backup::Info->last(): strType = incr +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Backup::Info->list(): strFilter = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strOrder = reverse +P00 DEBUG: Backup::Info->list=>: stryBackup = ([BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->last=>: strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->dbHistoryList=>: hDbHash = [hash] +P00 DEBUG: Backup::Info->confirmDb=>: bConfirmDb = true +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-INCR-2], [BACKUP-FULL-2]) +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-INCR-2]/backup.manifest.copy +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /[BACKUP-INCR-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-INCR-2]/backup.manifest +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-INCR-2]/backup.manifest.copy +P00 DEBUG: Protocol::Storage::Remote->exists(): strPathExp = [TEST_PATH]/db-master/db/base/postmaster.pid +P00 DEBUG: Protocol::Storage::Remote->exists=>: bExists = false +P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = [undef], hDatabaseMap = [undef], hTablespaceMap = [undef], oLastManifest = [object], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [undef], strLevel = [undef], strParentPath = [undef], strPath = [TEST_PATH]/db-master/db/base +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base/pg_tblspc +P00 DEBUG: Manifest->build: found tablespace 1 in offline mode +P00 DEBUG: Manifest->build: found tablespace 11 in offline mode +P00 DEBUG: Manifest->build: found tablespace 2 in offline mode +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/base +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = true, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [TS_PATH-1], strLevel = pg_tblspc/1, strParentPath = [TEST_PATH]/db-master/db/base/pg_tblspc, strPath = [TEST_PATH]/db-master/db/tablespace/ts1 +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/tablespace/ts1 +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = true, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [TS_PATH-1], strLevel = pg_tblspc/11, strParentPath = [TEST_PATH]/db-master/db/base/pg_tblspc/pg_tblspc, strPath = [TEST_PATH]/db-master/db/tablespace/ts11 +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/tablespace/ts11 +P00 DEBUG: Manifest->build(): bOnline = false, bTablespace = true, hDatabaseMap = [undef], hTablespaceMap = [hash], oLastManifest = [undef], oStorageDbMaster = [object], strDbVersion = 9.4, strFilter = [TS_PATH-1], strLevel = pg_tblspc/2, strParentPath = [TEST_PATH]/db-master/db/base/pg_tblspc/pg_tblspc/pg_tblspc, strPath = [TEST_PATH]/db-master/db/tablespace/ts2 +P00 DEBUG: Protocol::Storage::Remote->manifest(): strPathExp = [TEST_PATH]/db-master/db/tablespace/ts2 +P00 WARN: aborted backup [BACKUP-INCR-2] of same type exists, will be cleaned to remove invalid files and resumed +P00 TEST: PgBaCkReStTeSt-BACKUP-RESUME-PgBaCkReStTeSt +P00 DEBUG: Backup::Backup->resumeClean(): oAbortedManifest = [object], oManifest = [object], oStorageRepo = [object], strBackupLabel = [BACKUP-INCR-2] +P00 DETAIL: clean resumed backup path: /backup/db/[BACKUP-INCR-2] +P00 DEBUG: Storage::Local->manifest(): strPathExp = /[BACKUP-INCR-2] +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-INCR-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = false +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-INCR-2]/backup.manifest.copy +P00 DEBUG: Backup::Backup->processManifest(): bCompress = false, bHardLink = false, oBackupManifest = [object], strBackupLabel = [BACKUP-INCR-2], strDbCopyPath = [TEST_PATH]/db-master/db/base, strDbMasterPath = [TEST_PATH]/db-master/db/base, strDbVersion = 9.4, strLsnStart = [undef], strType = incr +P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = 1, strBackRestBin = [undef], strCommand = , strRemoteType = db +P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol +P00 DEBUG: Protocol::Local::Process->new(): bConfessError = , iSelectTimeout = <30>, strBackRestBin = <[BACKREST-BIN]>, strHostType = db +P00 DEBUG: Protocol::Local::Process->hostAdd(): iHostConfigIdx = 1, iProcessMax = 1 +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/32768/33001 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/32768/33000 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/16384/17000 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/global/pg_control to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/1/12000 to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/postgresql.conf to [BACKUP-FULL-2] +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/badchecksum.txt, pg_data/badchecksum.txt, 11, bogus, 0, [BACKUP-INCR-2], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_data/badchecksum.txt, strOp = backupFile, strQueue = pg_data +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt, pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt, 7, [undef], 1, [BACKUP-INCR-2], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt, strOp = backupFile, strQueue = pg_tblspc/2 +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt, pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt, 7, d85de07d6421d90aa9191c11c889bfde43680f0f, 1, [BACKUP-INCR-2], 0, [MODIFICATION-TIME-1], 1, {iWalId => 65535, iWalOffset => 65535}), strKey = pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt, strOp = backupFile, strQueue = pg_tblspc/1 +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2] +P00 DEBUG: Protocol::Local::Process->queueJob(): iHostConfigIdx = 1, rParam = ([TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init, pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init, 4, [undef], 0, [BACKUP-INCR-2], 0, [MODIFICATION-TIME-1], 1, [undef]), strKey = pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init, strOp = backupFile, strQueue = pg_tblspc/2 +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2] +P00 DEBUG: Backup::Backup->processManifest: reference pg_data/PG_VERSION to [BACKUP-FULL-2] +P00 DEBUG: Protocol::Local::Process->hostConnect: start local process: iHostConfigIdx = 1, iHostIdx = 0, iHostProcessIdx = 0, iProcessId = 1, strHostType = db +P00 DEBUG: Protocol::Local::Master->new(): iProcessIdx = 1, strCommand = [BACKREST-BIN] --command=backup --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --host-id=1 --lock-path=[TEST_PATH]/backup/lock --log-path=[TEST_PATH]/backup/log --process=1 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db local +P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --command=backup --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --host-id=1 --lock-path=[TEST_PATH]/backup/lock --log-path=[TEST_PATH]/backup/log --process=1 --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --type=db local, strId = 'local-1', strName = local +P00 DEBUG: Protocol::Local::Process->hostConnect=>: bResult = true +P00 DEBUG: Protocol::Local::Process->init: init local process: iDirection = 1, iHostIdx = 0, iProcessId = 1, iQueueIdx = 0, iQueueLastIdx = 2 +P00 DEBUG: Protocol::Local::Process->init=>: bResult = true +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_data/badchecksum.txt, strQueueIdx = 0 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (2, 11, 11, f927212cd08d11a42a666b2f04235398e9ceeb51, [undef]), strKey = pg_data/badchecksum.txt +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt, strQueueIdx = 1 +P00 WARN: resumed backup file pg_data/badchecksum.txt should have checksum bogus but actually has checksum f927212cd08d11a42a666b2f04235398e9ceeb51. The file will be recopied and backup will continue but this may be an issue unless the backup temp path is known to be corrupted. +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/badchecksum.txt (11B, 37%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 7, 7, dc7f76e43c46101b47acc55ae4d593a9e6983578, {bAlign => 0, bValid => 0}), strKey = pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init, strQueueIdx = 1 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 62%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192 +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (1, 4, 4, bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b, [undef]), strKey = pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init +P00 DEBUG: Protocol::Local::Process->process: get job from queue: iHostIdx = 0, iProcessId = 1, strKey = pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt, strQueueIdx = 2 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 75%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P00 DEBUG: Protocol::Local::Process->process: job complete: iProcessId = 1, rResult = (0, 7, 7, d85de07d6421d90aa9191c11c889bfde43680f0f, [undef]), strKey = pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt +P00 DEBUG: Protocol::Local::Process->process: no jobs found, stop local: iHostConfigIdx = [undef], iHostIdx = 0, iProcessId = 1, strHostType = [undef] +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 100%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f +P00 DEBUG: Protocol::Local::Process->process: all jobs complete +P00 DEBUG: Backup::Backup->processManifest=>: lSizeTotal = 29 +P00 INFO: incr backup size = 29B +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = db +P00 DEBUG: Protocol::Command::Master->close=>: iExitStatus = 0 +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Storage::Local->pathSync(): bRecurse = true, strPathExp = /[BACKUP-INCR-2] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-INCR-2]/backup.manifest +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db/[BACKUP-INCR-2] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/[BACKUP-INCR-2]/backup.manifest.copy +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db/[BACKUP-INCR-2] +P00 INFO: new backup label = [BACKUP-INCR-2] +P00 DEBUG: Storage::Local->openWrite(): bAtomic = true, bPathCreate = true, lTimestamp = [undef], rhyFilter = ({strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup.history/[YEAR-1]/[BACKUP-INCR-2].manifest.gz +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = /[BACKUP-INCR-2]/backup.manifest +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = /backup/db/[BACKUP-INCR-2]/backup.manifest +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup.history +P00 DEBUG: Storage::Local->remove(): bIgnoreMissing = , bRecurse = , strPathFileExp = /latest +P00 DEBUG: Storage::Local->remove=>: bRemoved = true +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db +P00 DEBUG: Storage::Local->openWrite(): bAtomic = , bPathCreate = , lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /backup/db/backup.info.copy +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = /backup/db +P00 DEBUG: Storage::Local->pathSync(): bRecurse = , strPathExp = +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 DEBUG: Backup::Info->new(): bIgnoreMissing = , bLoad = , bRequired = , bValidate = , oStorage = <[object]>, strBackupClusterPath = /backup/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /backup/db/backup.info +P00 DEBUG: Backup::Info->reconstruct(): bRequired = , bSave = , iCatalogVersion = [undef], iControlVersion = [undef], strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = /backup/db, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-FULL-2], [BACKUP-INCR-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-INCR-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-FULL-2] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-FULL-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Storage::Local->pathExists(): strPathExp = /backup/db/[BACKUP-INCR-2] +P00 DEBUG: Storage::Local->pathExists=>: bExists = true +P00 DEBUG: Storage::Local->exists(): strFileExp = /backup/db/[BACKUP-INCR-2]/backup.manifest +P00 DEBUG: Storage::Local->exists=>: bExists = true +P00 DEBUG: Backup::Common::backupRegExpGet(): bAnchor = , bDifferential = true, bFull = true, bIncremental = true +P00 DEBUG: Backup::Common::backupRegExpGet=>: strRegExp = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$ +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = , strExpression = ^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$, strPathExp = , strSortOrder = reverse +P00 DEBUG: Storage::Local->list=>: stryFileList = ([BACKUP-INCR-2], [BACKUP-FULL-2]) +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-INCR-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 DEBUG: Backup::Info->current(): strBackup = [BACKUP-FULL-2] +P00 DEBUG: Backup::Info->current=>: bTest = true +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: expire command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-INCR-2]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-INCR-2]" +backup-prior="[BACKUP-FULL-2]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="incr" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base","type":"path"} +pg_tblspc/1={"path":"[TEST_PATH]/db-master/db/tablespace/ts1","tablespace-id":"1","tablespace-name":"ts1","type":"link"} +pg_tblspc/11={"path":"[TEST_PATH]/db-master/db/tablespace/ts11","tablespace-id":"11","tablespace-name":"ts11","type":"link"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"e0101dd8ffb910c9c202ca35b5f828bcb9697bed","checksum-page":false,"checksum-page-error":[1],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt={"checksum":"d85de07d6421d90aa9191c11c889bfde43680f0f","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/1={"destination":"[TEST_PATH]/db-master/db/tablespace/ts1"} +pg_data/pg_tblspc/11={"destination":"[TEST_PATH]/db-master/db/tablespace/ts11"} +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/1={} +pg_tblspc/1/[TS_PATH-1]={} +pg_tblspc/1/[TS_PATH-1]/16384={} +pg_tblspc/11={} +pg_tblspc/11/[TS_PATH-1]={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +diff backup - cannot resume - new diff (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --type=diff --stanza=db backup --test --test-delay=0.2 --test-point=backup-noresume=y +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --test --test-delay=0.2 --test-point=backup-noresume=y --type=diff +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 WARN: backup [BACKUP-INCR-2] missing manifest removed from backup.info +P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] +P00 WARN: aborted backup [BACKUP-INCR-2] cannot be resumed: new backup-type 'diff' does not match aborted backup-type 'incr' +P00 TEST: PgBaCkReStTeSt-BACKUP-NORESUME-PgBaCkReStTeSt +P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/badchecksum.txt (11B, 37%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 62%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 75%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 100%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192 +P00 INFO: diff backup size = 29B +P00 INFO: new backup label = [BACKUP-DIFF-1] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-DIFF-1]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-DIFF-1]" +backup-prior="[BACKUP-FULL-2]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="diff" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base","type":"path"} +pg_tblspc/1={"path":"[TEST_PATH]/db-master/db/tablespace/ts1","tablespace-id":"1","tablespace-name":"ts1","type":"link"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"e0101dd8ffb910c9c202ca35b5f828bcb9697bed","checksum-page":false,"checksum-page-error":[1],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt={"checksum":"d85de07d6421d90aa9191c11c889bfde43680f0f","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/1={"destination":"[TEST_PATH]/db-master/db/tablespace/ts1"} +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/1={} +pg_tblspc/1/[TS_PATH-1]={} +pg_tblspc/1/[TS_PATH-1]/16384={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-1]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +diff backup - cannot resume - disabled / no repo link (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --no-resume --log-level-console=detail --type=diff --stanza=db backup --test --test-delay=0.2 --test-point=backup-noresume=y +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --no-resume --stanza=db --start-fast --test --test-delay=0.2 --test-point=backup-noresume=y --type=diff +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 WARN: backup [BACKUP-DIFF-1] missing manifest removed from backup.info +P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] +P00 WARN: aborted backup [BACKUP-DIFF-1] cannot be resumed: resume is disabled +P00 TEST: PgBaCkReStTeSt-BACKUP-NORESUME-PgBaCkReStTeSt +P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/badchecksum.txt (11B, 37%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 62%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 75%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 100%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192 +P00 INFO: diff backup size = 29B +P00 INFO: new backup label = [BACKUP-DIFF-2] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-DIFF-2]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-DIFF-2]" +backup-prior="[BACKUP-FULL-2]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="diff" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base","type":"path"} +pg_tblspc/1={"path":"[TEST_PATH]/db-master/db/tablespace/ts1","tablespace-id":"1","tablespace-name":"ts1","type":"link"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"e0101dd8ffb910c9c202ca35b5f828bcb9697bed","checksum-page":false,"checksum-page-error":[1],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt={"checksum":"d85de07d6421d90aa9191c11c889bfde43680f0f","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/1={"destination":"[TEST_PATH]/db-master/db/tablespace/ts1"} +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/1={} +pg_tblspc/1/[TS_PATH-1]={} +pg_tblspc/1/[TS_PATH-1]/16384={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +restore, backup '[BACKUP-DIFF-2]', expect exit 40 - fail on used path (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --set=[BACKUP-DIFF-2] --log-level-console=detail --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --set=[BACKUP-DIFF-2] --stanza=db +P00 INFO: restore backup set [BACKUP-DIFF-2] +P00 DETAIL: check [TEST_PATH]/db-master/db/base exists +P00 ERROR: [040]: cannot restore to path '[TEST_PATH]/db-master/db/base' that contains files - try using --delta if this is what you intended +P00 INFO: restore command end: aborted with exception [040] + +restore, backup '[BACKUP-DIFF-2]', remap - remap all paths (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --set=[BACKUP-DIFF-2] --log-level-console=detail --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --set=[BACKUP-DIFF-2] --stanza=db --tablespace-map=1=[TEST_PATH]/db-master/db/tablespace/ts1-2 --tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 +P00 INFO: restore backup set [BACKUP-DIFF-2] +P00 INFO: remap $PGDATA directory to [TEST_PATH]/db-master/db/base-2 +P00 INFO: remap tablespace pg_tblspc/1 directory to [TEST_PATH]/db-master/db/tablespace/ts1-2 +P00 INFO: remap tablespace pg_tblspc/2 directory to [TEST_PATH]/db-master/db/tablespace/ts2-2 +P00 DETAIL: check [TEST_PATH]/db-master/db/base-2 exists +P00 DETAIL: check [TEST_PATH]/db-master/db/tablespace/ts1-2 exists +P00 DETAIL: check [TEST_PATH]/db-master/db/tablespace/ts2-2 exists +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 39%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/32768/33000.32767 (32KB, 59%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/32768/33000 (32KB, 79%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/16384/17000 (16KB, 89%) checksum e0101dd8ffb910c9c202ca35b5f828bcb9697bed +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/global/pg_control.pgbackrest.tmp (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/1/12000 (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/postgresql.conf (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/badchecksum.txt (11B, 99%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/pg_stat/global.stat (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/32768/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/16384/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/1/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 99%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 99%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 100%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f +P00 INFO: write [TEST_PATH]/db-master/db/base-2/recovery.conf +P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started) +P00 INFO: restore command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/db/base-2/recovery.conf +------------------------------------------------------------------ +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-get %f "%p"' + +restore delta, backup '[BACKUP-DIFF-2]', remap - ensure file in tblspc root remains after --delta (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-DIFF-2] --log-level-console=detail --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --set=[BACKUP-DIFF-2] --stanza=db --tablespace-map=1=[TEST_PATH]/db-master/db/tablespace/ts1-2 --tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 +P00 INFO: restore backup set [BACKUP-DIFF-2] +P00 INFO: remap $PGDATA directory to [TEST_PATH]/db-master/db/base-2 +P00 INFO: remap tablespace pg_tblspc/1 directory to [TEST_PATH]/db-master/db/tablespace/ts1-2 +P00 INFO: remap tablespace pg_tblspc/2 directory to [TEST_PATH]/db-master/db/tablespace/ts2-2 +P00 DETAIL: check [TEST_PATH]/db-master/db/base-2 exists +P00 DETAIL: check [TEST_PATH]/db-master/db/tablespace/ts1-2 exists +P00 DETAIL: check [TEST_PATH]/db-master/db/tablespace/ts2-2 exists +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/tablespace/ts2-2/[TS_PATH-1] +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/tablespace/ts1-2/[TS_PATH-1] +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/base-2 +P00 DETAIL: preserve file [TEST_PATH]/db-master/db/base-2/recovery.conf +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/32768/33001 - exists and matches backup (64KB, 39%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/32768/33000.32767 - exists and matches backup (32KB, 59%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/32768/33000 - exists and matches backup (32KB, 79%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/16384/17000 - exists and matches backup (16KB, 89%) checksum e0101dd8ffb910c9c202ca35b5f828bcb9697bed +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/global/pg_control.pgbackrest.tmp (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/1/12000 - exists and matches backup (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/postgresql.conf - exists and matches backup (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/badchecksum.txt - exists and matches backup (11B, 99%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/pg_stat/global.stat - exists and matches backup (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/32768/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/16384/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/1/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt - exists and matches backup (7B, 99%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init - exists and matches backup (4B, 99%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt - exists and matches backup (7B, 100%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f +P00 INFO: write [TEST_PATH]/db-master/db/base-2/recovery.conf +P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started) +P00 INFO: restore command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/db/base-2/recovery.conf +------------------------------------------------------------------ +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-get %f "%p"' + +incr backup - add files and remove tablespace 2 (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-DIFF-2], version = [VERSION-1] +P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-DIFF-2] +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt (8B, 61%) checksum e324463005236d83e6e54795dbddd20a74533bf3 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt: file size 8 is not divisible by page size 8192 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base2.txt (5B, 100%) checksum 09b5e31766be1dba1ec27de82f975c1b6eea2a92 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/base/base2.txt: file size 5 is not divisible by page size 8192 +P00 INFO: incr backup size = 13B +P00 INFO: new backup label = [BACKUP-INCR-3] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base-2 + +[db:restore] +tablespace-map=1=[TEST_PATH]/db-master/db/tablespace/ts1-2 +tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base-2 +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-INCR-3]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-INCR-3]" +backup-prior="[BACKUP-DIFF-2]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="incr" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base-2","type":"path"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2-2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"reference":"[BACKUP-DIFF-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"e0101dd8ffb910c9c202ca35b5f828bcb9697bed","checksum-page":false,"checksum-page-error":[1],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/base2.txt={"checksum":"09b5e31766be1dba1ec27de82f975c1b6eea2a92","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","reference":"[BACKUP-DIFF-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","checksum-page":false,"reference":"[BACKUP-DIFF-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt={"checksum":"e324463005236d83e6e54795dbddd20a74533bf3","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2-2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-2]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +incr backup - update files - fail on missing backup.info (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 ERROR: [055]: unable to open /backup/db/backup.info or /backup/db/backup.info.copy +P00 ERROR: [055]: /backup/db/backup.info does not exist and is required to perform a backup. + HINT: has a stanza-create been performed? +P00 INFO: backup command end: aborted with exception [055] + +stanza-create db - fail on backup directory missing backup.info (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 ERROR: [055]: backup information missing + HINT: use stanza-create --force to force the stanza data to be created. +P00 INFO: stanza-create command end: aborted with exception [055] + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - create required data for stanza (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-2]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +incr backup - update files (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-INCR-3], version = [VERSION-1] +P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-INCR-3] +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/16384/17000 (8B, 100%) checksum 9a53d532e27785e681766c98516a5e93f096a501 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/base/16384/17000: file size 8 is not divisible by page size 8192 +P00 INFO: incr backup size = 8B +P00 INFO: new backup label = [BACKUP-INCR-4] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base-2 + +[db:restore] +tablespace-map=1=[TEST_PATH]/db-master/db/tablespace/ts1-2 +tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base-2 +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-INCR-4]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-INCR-4]" +backup-prior="[BACKUP-INCR-3]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="incr" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base-2","type":"path"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2-2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"reference":"[BACKUP-DIFF-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"9a53d532e27785e681766c98516a5e93f096a501","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/base2.txt={"checksum":"09b5e31766be1dba1ec27de82f975c1b6eea2a92","checksum-page":false,"reference":"[BACKUP-INCR-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","reference":"[BACKUP-DIFF-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","checksum-page":false,"reference":"[BACKUP-DIFF-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt={"checksum":"e324463005236d83e6e54795dbddd20a74533bf3","checksum-page":false,"reference":"[BACKUP-INCR-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2-2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-2]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-4]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-INCR-3]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]","[BACKUP-INCR-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +diff backup - updates since last full (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --type=diff --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=diff +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] +P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/badchecksum.txt (11B, 25%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/16384/17000 (8B, 44%) checksum 9a53d532e27785e681766c98516a5e93f096a501 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/base/16384/17000: file size 8 is not divisible by page size 8192 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base2.txt (5B, 55%) checksum 09b5e31766be1dba1ec27de82f975c1b6eea2a92 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/base/base2.txt: file size 5 is not divisible by page size 8192 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt (8B, 74%) checksum e324463005236d83e6e54795dbddd20a74533bf3 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt: file size 8 is not divisible by page size 8192 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 90%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 100%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P00 INFO: diff backup size = 43B +P00 INFO: new backup label = [BACKUP-DIFF-3] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base-2 + +[db:restore] +tablespace-map=1=[TEST_PATH]/db-master/db/tablespace/ts1-2 +tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base-2 +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-DIFF-3]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-DIFF-3]" +backup-prior="[BACKUP-FULL-2]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="diff" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base-2","type":"path"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2-2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"9a53d532e27785e681766c98516a5e93f096a501","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/base2.txt={"checksum":"09b5e31766be1dba1ec27de82f975c1b6eea2a92","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt={"checksum":"e324463005236d83e6e54795dbddd20a74533bf3","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2-2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-2]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-4]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-INCR-3]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]","[BACKUP-INCR-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +incr backup - remove files - but won't affect manifest (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --stanza=db backup --test --test-delay=1 --test-point=manifest-build=y +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --test --test-delay=1 --test-point=manifest-build=y +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-DIFF-3], version = [VERSION-1] +P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-DIFF-3] +P00 TEST: PgBaCkReStTeSt-MANIFEST-BUILD-PgBaCkReStTeSt +P00 INFO: incr backup size = 0B +P00 INFO: new backup label = [BACKUP-INCR-5] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base-2 + +[db:restore] +tablespace-map=1=[TEST_PATH]/db-master/db/tablespace/ts1-2 +tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base-2 +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-INCR-5]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-INCR-5]" +backup-prior="[BACKUP-DIFF-3]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="incr" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base-2","type":"path"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2-2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"reference":"[BACKUP-DIFF-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"9a53d532e27785e681766c98516a5e93f096a501","checksum-page":false,"reference":"[BACKUP-DIFF-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/base2.txt={"checksum":"09b5e31766be1dba1ec27de82f975c1b6eea2a92","checksum-page":false,"reference":"[BACKUP-DIFF-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","reference":"[BACKUP-DIFF-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","checksum-page":false,"reference":"[BACKUP-DIFF-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt={"checksum":"e324463005236d83e6e54795dbddd20a74533bf3","checksum-page":false,"reference":"[BACKUP-DIFF-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2-2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-2]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-4]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-INCR-3]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]","[BACKUP-INCR-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-5]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-3]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +diff backup - remove files during backup (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --type=diff --stanza=db backup --test --test-delay=1 --test-point=manifest-build=y +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --test --test-delay=1 --test-point=manifest-build=y --type=diff +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1] +P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2] +P00 TEST: PgBaCkReStTeSt-MANIFEST-BUILD-PgBaCkReStTeSt +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/badchecksum.txt (11B, 31%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P00 DETAIL: skip file removed by database db-master:[TEST_PATH]/db-master/db/base-2/base/base2.txt +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt (12B, 68%) checksum dfcb8679956b734706cf87259d50c88f83e80e66 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt: file size 12 is not divisible by page size 8192 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 88%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 100%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P00 INFO: diff backup size = 35B +P00 INFO: new backup label = [BACKUP-DIFF-4] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base-2 + +[db:restore] +tablespace-map=1=[TEST_PATH]/db-master/db/tablespace/ts1-2 +tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base-2 +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-DIFF-4]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-DIFF-4]" +backup-prior="[BACKUP-FULL-2]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="diff" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=true +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base-2","type":"path"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2-2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","checksum-page":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","checksum-page":false,"checksum-page-error":[0,[3,5],7],"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-2]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt={"checksum":"dfcb8679956b734706cf87259d50c88f83e80e66","checksum-page":false,"size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2-2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-2]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-4]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-INCR-3]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]","[BACKUP-INCR-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-5]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-3]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-4]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +full backup - update file (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=full +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 44%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/32768/33000.32767 (32KB, 66%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/32768/33000 (32KB, 88%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/global/pg_control (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/1/12000 (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/postgresql.conf (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/badchecksum.txt (11B, 99%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/16384/17000 (9B, 99%) checksum 7579ada0808d7f98087a0a586d0df9de009cdc33 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_stat/global.stat (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/32768/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/16384/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/1/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt (12B, 99%) checksum dfcb8679956b734706cf87259d50c88f83e80e66 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 99%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 100%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P00 INFO: full backup size = 144KB +P00 INFO: new backup label = [BACKUP-FULL-3] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base-2 + +[db:restore] +tablespace-map=1=[TEST_PATH]/db-master/db/tablespace/ts1-2 +tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base-2 +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-FULL-3]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-FULL-3]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="full" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=false +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base-2","type":"path"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2-2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"7579ada0808d7f98087a0a586d0df9de009cdc33","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt={"checksum":"dfcb8679956b734706cf87259d50c88f83e80e66","size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2-2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-2]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-2]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-4]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-INCR-3]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-2]","[BACKUP-INCR-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-INCR-5]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-DIFF-3]","backup-reference":["[BACKUP-FULL-2]","[BACKUP-DIFF-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"incr","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-4]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-2]","backup-reference":["[BACKUP-FULL-2]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":true,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-FULL-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +info db stanza - normal output (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --stanza=db info +------------------------------------------------------------------------------------------------------------------------------------ +stanza: db + status: ok + wal archive min/max: none present + + full backup: [BACKUP-FULL-2] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 160KB, backup size: 160KB + repository size: 160KB, repository backup size: 160KB + + diff backup: [BACKUP-DIFF-2] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 160KB, backup size: 29B + repository size: 160KB, repository backup size: 29B + backup reference list: [BACKUP-FULL-2] + + incr backup: [BACKUP-INCR-3] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 160KB, backup size: 13B + repository size: 160KB, repository backup size: 13B + backup reference list: [BACKUP-FULL-2], [BACKUP-DIFF-2] + + incr backup: [BACKUP-INCR-4] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 144KB, backup size: 8B + repository size: 144KB, repository backup size: 8B + backup reference list: [BACKUP-FULL-2], [BACKUP-DIFF-2], [BACKUP-INCR-3] + + diff backup: [BACKUP-DIFF-3] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 144KB, backup size: 43B + repository size: 144KB, repository backup size: 43B + backup reference list: [BACKUP-FULL-2] + + incr backup: [BACKUP-INCR-5] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 144KB, backup size: 0B + repository size: 144KB, repository backup size: 0B + backup reference list: [BACKUP-FULL-2], [BACKUP-DIFF-3] + + diff backup: [BACKUP-DIFF-4] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 144KB, backup size: 34B + repository size: 144KB, repository backup size: 34B + backup reference list: [BACKUP-FULL-2] + + full backup: [BACKUP-FULL-3] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 144KB, backup size: 144KB + repository size: 144KB, repository backup size: 144KB + +info db stanza - normal output (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=warn --stanza=db --output=json info +------------------------------------------------------------------------------------------------------------------------------------ +[ + { + "archive" : { + "max" : null, + "min" : null + }, + "backup" : [ + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-FULL-2]", + "prior" : null, + "reference" : null, + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "full" + }, + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-DIFF-2]", + "prior" : "[BACKUP-FULL-2]", + "reference" : [ + "[BACKUP-FULL-2]" + ], + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "diff" + }, + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-INCR-3]", + "prior" : "[BACKUP-DIFF-2]", + "reference" : [ + "[BACKUP-FULL-2]", + "[BACKUP-DIFF-2]" + ], + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "incr" + }, + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-INCR-4]", + "prior" : "[BACKUP-INCR-3]", + "reference" : [ + "[BACKUP-FULL-2]", + "[BACKUP-DIFF-2]", + "[BACKUP-INCR-3]" + ], + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "incr" + }, + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-DIFF-3]", + "prior" : "[BACKUP-FULL-2]", + "reference" : [ + "[BACKUP-FULL-2]" + ], + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "diff" + }, + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-INCR-5]", + "prior" : "[BACKUP-DIFF-3]", + "reference" : [ + "[BACKUP-FULL-2]", + "[BACKUP-DIFF-3]" + ], + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "incr" + }, + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-DIFF-4]", + "prior" : "[BACKUP-FULL-2]", + "reference" : [ + "[BACKUP-FULL-2]" + ], + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "diff" + }, + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-FULL-3]", + "prior" : null, + "reference" : null, + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "full" + } + ], + "db" : [ + { + "id" : "1", + "system-id" : 6353949018581704918, + "version" : "9.4" + } + ], + "name" : "db", + "status" : { + "code" : 0, + "message" : "ok" + } + } +] + +expire full=1 (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --retention-full=1 --stanza=db expire +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --retention-full=1 --stanza=db +P00 INFO: expire full backup set: [BACKUP-FULL-2], [BACKUP-DIFF-2], [BACKUP-INCR-3], [BACKUP-INCR-4], [BACKUP-DIFF-3], [BACKUP-INCR-5], [BACKUP-DIFF-4] +P00 INFO: remove expired backup [BACKUP-DIFF-4] +P00 INFO: remove expired backup [BACKUP-DIFF-3] +P00 INFO: remove expired backup [BACKUP-INCR-4] +P00 INFO: remove expired backup [BACKUP-INCR-3] +P00 INFO: remove expired backup [BACKUP-DIFF-2] +P00 INFO: remove expired backup [BACKUP-FULL-2] +P00 INFO: expire command end: completed successfully + +diff backup - add file (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --checksum-page --type=diff --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --checksum-page --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=diff +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1] +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0 +P00 INFO: diff backup size = 9B +P00 INFO: new backup label = [BACKUP-DIFF-5] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base-2 + +[db:restore] +tablespace-map=1=[TEST_PATH]/db-master/db/tablespace/ts1-2 +tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base-2 +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-DIFF-5]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-DIFF-5]" +backup-prior="[BACKUP-FULL-3]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="diff" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=false +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base-2","type":"path"} +pg_tblspc/2={"path":"[TEST_PATH]/db-master/db/tablespace/ts2-2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"7579ada0808d7f98087a0a586d0df9de009cdc33","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/base2.txt={"checksum":"cafac3c59553f2cfde41ce2e62e7662295f108c0","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt={"checksum":"dfcb8679956b734706cf87259d50c88f83e80e66","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/2={"destination":"[TEST_PATH]/db-master/db/tablespace/ts2-2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-5]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-3]","backup-reference":["[BACKUP-FULL-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +restore delta, remap - selective restore 16384 (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --log-level-console=detail --db-include=16384 --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-include=16384=1 --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db --tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 +P00 INFO: restore backup set [BACKUP-DIFF-5] +P00 INFO: remap tablespace pg_tblspc/2 directory to [TEST_PATH]/db-master/db/tablespace/ts2-2 +P00 DETAIL: check [TEST_PATH]/db-master/db/base-2 exists +P00 DETAIL: check [TEST_PATH]/db-master/db/tablespace/ts2-2 exists +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/tablespace/ts2-2/[TS_PATH-1] +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/base-2 +P00 DETAIL: preserve file [TEST_PATH]/db-master/db/base-2/recovery.conf +P00 DETAIL: databases for include/exclude (1, 16384, 32768) +P00 DETAIL: database filter: (^pg_data\/base\/32768\/)|(^pg_tblspc/2\/[TS_PATH-1]\/32768\/) +P01 DETAIL: restore zeroed file [TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 44%) +P01 DETAIL: restore zeroed file [TEST_PATH]/db-master/db/base-2/base/32768/33000.32767 (32KB, 66%) +P01 DETAIL: restore zeroed file [TEST_PATH]/db-master/db/base-2/base/32768/33000 (32KB, 88%) +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/global/pg_control.pgbackrest.tmp (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/1/12000 - exists and matches backup (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/postgresql.conf - exists and matches backup (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/badchecksum.txt - exists and matches backup (11B, 99%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/base2.txt - exists and matches backup (9B, 99%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/16384/17000 - exists and matches backup (9B, 99%) checksum 7579ada0808d7f98087a0a586d0df9de009cdc33 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/pg_stat/global.stat - exists and matches backup (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/32768/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/16384/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/1/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore zeroed file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt (12B, 99%) +P01 DETAIL: restore zeroed file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 99%) +P01 DETAIL: restore zeroed file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 100%) +P00 INFO: write [TEST_PATH]/db-master/db/base-2/recovery.conf +P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started) +P00 INFO: restore command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/db/base-2/recovery.conf +------------------------------------------------------------------ +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-get %f "%p"' + +restore delta, remap - selective restore 32768 (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --log-level-console=detail --db-include=32768 --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-include=32768=1 --db-path=[TEST_PATH]/db-master/db/base-2 --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db --tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2 +P00 INFO: restore backup set [BACKUP-DIFF-5] +P00 INFO: remap tablespace pg_tblspc/2 directory to [TEST_PATH]/db-master/db/tablespace/ts2-2 +P00 DETAIL: check [TEST_PATH]/db-master/db/base-2 exists +P00 DETAIL: check [TEST_PATH]/db-master/db/tablespace/ts2-2 exists +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/tablespace/ts2-2/[TS_PATH-1] +P00 INFO: remove invalid files/paths/links from [TEST_PATH]/db-master/db/base-2 +P00 DETAIL: preserve file [TEST_PATH]/db-master/db/base-2/recovery.conf +P00 DETAIL: databases for include/exclude (1, 16384, 32768) +P00 DETAIL: database filter: (^pg_data\/base\/16384\/)|(^pg_tblspc/2\/[TS_PATH-1]\/16384\/) +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 44%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/32768/33000.32767 (32KB, 66%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/32768/33000 (32KB, 88%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/global/pg_control.pgbackrest.tmp (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/1/12000 - exists and matches backup (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/postgresql.conf - exists and matches backup (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/badchecksum.txt - exists and matches backup (11B, 99%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/base2.txt - exists and matches backup (9B, 99%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0 +P01 DETAIL: restore zeroed file [TEST_PATH]/db-master/db/base-2/base/16384/17000 (9B, 99%) +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/pg_stat/global.stat - exists and matches backup (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/32768/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/16384/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/base/1/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 DETAIL: restore file [TEST_PATH]/db-master/db/base-2/PG_VERSION - exists and matches backup (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt (12B, 99%) checksum dfcb8679956b734706cf87259d50c88f83e80e66 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 99%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 100%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P00 INFO: write [TEST_PATH]/db-master/db/base-2/recovery.conf +P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started) +P00 INFO: restore command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/db/base-2/recovery.conf +------------------------------------------------------------------ +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-get %f "%p"' + +restore delta, remap, expect exit 80 - error on invalid id (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --log-level-console=warn --db-include=7777 --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 ERROR: [080]: database to include '7777' does not exist + +restore delta, remap, expect exit 81 - error on system id (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --log-level-console=warn --db-include=1 --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 ERROR: [081]: system databases (template0, postgres, etc.) are included by default + +restore, remap, expect exit 73 - no tablespace remap - error when tablespace dir does not exist (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --tablespace-map-all=../../tablespace --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base-2/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db --tablespace-map-all=../../tablespace +P00 INFO: restore backup set [BACKUP-DIFF-5] +P00 INFO: remap $PGDATA directory to [TEST_PATH]/db-master/db/base-2/base +P00 INFO: remap tablespace pg_tblspc/2 directory to ../../tablespace/ts2 +P00 DETAIL: check [TEST_PATH]/db-master/db/base-2/base exists +P00 DETAIL: check [TEST_PATH]/db-master/db/base-2/tablespace exists +P00 ERROR: [073]: cannot restore to missing path [TEST_PATH]/db-master/db/base-2/tablespace +P00 INFO: restore command end: aborted with exception [073] + +restore - no tablespace remap (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --tablespace-map-all=../../tablespace --log-level-console=detail --stanza=db restore +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: restore command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --no-compress --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base-2/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db --tablespace-map-all=../../tablespace +P00 INFO: restore backup set [BACKUP-DIFF-5] +P00 INFO: remap $PGDATA directory to [TEST_PATH]/db-master/db/base-2/base +P00 INFO: remap tablespace pg_tblspc/2 directory to ../../tablespace/ts2 +P00 DETAIL: check [TEST_PATH]/db-master/db/base-2/base exists +P00 DETAIL: check [TEST_PATH]/db-master/db/base-2/tablespace exists +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/base/32768/33001 (64KB, 44%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/base/32768/33000.32767 (32KB, 66%) checksum 21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/base/32768/33000 (32KB, 88%) checksum 4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/global/pg_control.pgbackrest.tmp (8KB, 94%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/base/1/12000 (8KB, 99%) checksum 22c98d248ff548311eda88559e4a8405ed77c003 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/postgresql.conf (21B, 99%) checksum 6721d92c9fcdf4248acff1f9a1377127d9064807 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/badchecksum.txt (11B, 99%) checksum f927212cd08d11a42a666b2f04235398e9ceeb51 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/base/base2.txt (9B, 99%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/base/16384/17000 (9B, 99%) checksum 7579ada0808d7f98087a0a586d0df9de009cdc33 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/pg_stat/global.stat (5B, 99%) checksum e350d5ce0153f3e22d5db21cf2a4eff00f3ee877 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/base/32768/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/base/16384/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/base/1/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt (12B, 99%) checksum dfcb8679956b734706cf87259d50c88f83e80e66 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 99%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578 +P01 INFO: restore file [TEST_PATH]/db-master/db/base-2/base/pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init (4B, 100%) checksum bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b +P00 INFO: write [TEST_PATH]/db-master/db/base-2/base/recovery.conf +P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started) +P00 INFO: restore command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/db/base-2/base/recovery.conf +----------------------------------------------------------------------- +restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-get %f "%p"' + +info all stanzas - normal output (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=warn info +------------------------------------------------------------------------------------------------------------------------------------ +stanza: db + status: ok + wal archive min/max: none present + + full backup: [BACKUP-FULL-3] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 144KB, backup size: 144KB + repository size: 144KB, repository backup size: 144KB + + diff backup: [BACKUP-DIFF-5] + timestamp start/stop: [TIMESTAMP-STR] + wal start/stop: n/a + database size: 144KB, backup size: 9B + repository size: 144KB, repository backup size: 9B + backup reference list: [BACKUP-FULL-3] + +info all stanzas - normal output (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --output=json info +------------------------------------------------------------------------------------------------------------------------------------ +[ + { + "archive" : { + "max" : null, + "min" : null + }, + "backup" : [ + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-FULL-3]", + "prior" : null, + "reference" : null, + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "full" + }, + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-DIFF-5]", + "prior" : "[BACKUP-FULL-3]", + "reference" : [ + "[BACKUP-FULL-3]" + ], + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "diff" + } + ], + "db" : [ + { + "id" : "1", + "system-id" : 6353949018581704918, + "version" : "9.4" + } + ], + "name" : "db", + "status" : { + "code" : 0, + "message" : "ok" + } + } +] + +info bogus stanza - bogus stanza (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=warn --stanza=bogus info +------------------------------------------------------------------------------------------------------------------------------------ +stanza: bogus + status: error (missing stanza path) + wal archive min/max: none present + +info bogus stanza - bogus stanza (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --stanza=bogus --output=json info +------------------------------------------------------------------------------------------------------------------------------------ +[ + { + "backup" : [], + "db" : [], + "name" : "bogus", + "status" : { + "code" : 1, + "message" : "missing stanza path" + } + } +] + +diff backup - config file not validated on remote (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --type=diff --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=diff +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1] +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0 +P00 INFO: diff backup size = 9B +P00 INFO: new backup label = [BACKUP-DIFF-6] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base-2/base + +[db:restore] + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 +bogus=bogus + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base-2/base +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-DIFF-6]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-DIFF-6]" +backup-prior="[BACKUP-FULL-3]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="diff" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=false +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base-2/base","type":"path"} +pg_tblspc/2={"path":"../../tablespace/ts2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"7579ada0808d7f98087a0a586d0df9de009cdc33","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/base2.txt={"checksum":"cafac3c59553f2cfde41ce2e62e7662295f108c0","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt={"checksum":"dfcb8679956b734706cf87259d50c88f83e80e66","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/2={"destination":"../../tablespace/ts2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-5]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-3]","backup-reference":["[BACKUP-FULL-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-6]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-3]","backup-reference":["[BACKUP-FULL-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +diff backup - option backup-standby reset - backup performed from master (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=detail --backup-standby --type=diff --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --backup-standby --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db --start-fast --type=diff +P00 WARN: option retention-full is not set, the repository may run out of space + HINT: to retain full backups indefinitely (without warning), set option 'retention-full' to the maximum. +P00 WARN: option backup-standby is enabled but standby is not properly configured - backups will be performed from the master +P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1] +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0 +P00 INFO: diff backup size = 9B +P00 INFO: new backup label = [BACKUP-DIFF-7] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --no-compress --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: option 'retention-archive' is not set - archive logs will not be expired +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base-2/base + +[db:restore] + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base-2/base +db-user=[USER-1] + +[global] +compress=n +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + ++ supplemental file: /backup/db/[BACKUP-DIFF-7]/backup.manifest +--------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup] +backup-label="[BACKUP-DIFF-7]" +backup-prior="[BACKUP-FULL-3]" +backup-timestamp-copy-start=[TIMESTAMP] +backup-timestamp-start=[TIMESTAMP] +backup-timestamp-stop=[TIMESTAMP] +backup-type="diff" + +[backup:db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[backup:option] +option-archive-check=true +option-archive-copy=true +option-backup-standby=false +option-checksum-page=false +option-compress=false +option-hardlink=false +option-online=false + +[backup:target] +pg_data={"path":"[TEST_PATH]/db-master/db/base-2/base","type":"path"} +pg_tblspc/2={"path":"../../tablespace/ts2","tablespace-id":"2","tablespace-name":"ts2","type":"link"} + +[target:file] +pg_data/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/badchecksum.txt={"checksum":"f927212cd08d11a42a666b2f04235398e9ceeb51","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/12000={"checksum":"22c98d248ff548311eda88559e4a8405ed77c003","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/1/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","mode":"0660","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/17000={"checksum":"7579ada0808d7f98087a0a586d0df9de009cdc33","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/16384/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000={"checksum":"4a383e4fb8b5cd2a4e8fab91ef63dce48e532a2f","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33000.32767={"checksum":"21e2c7c1a326682c07053b7d6a5a40dbd49c2ec5","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/33001={"checksum":"6bf316f11d28c28914ea9be92c00de9bea6d9a6b","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/32768/PG_VERSION={"checksum":"184473f470864e067ee3a22e64b47b0a1c356f29","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/base/base2.txt={"checksum":"cafac3c59553f2cfde41ce2e62e7662295f108c0","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_data/global/pg_control={"checksum":"89373d9f2973502940de06bc5212489df3f8a912","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/pg_stat/global.stat={"checksum":"e350d5ce0153f3e22d5db21cf2a4eff00f3ee877","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_data/postgresql.conf={"checksum":"6721d92c9fcdf4248acff1f9a1377127d9064807","master":true,"reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-2]} +pg_tblspc/2/[TS_PATH-1]/32768/pg_internal.init={"checksum":"bc46a4e0420d357db7bfbcb7b5fcbc613dc48c1b","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt={"checksum":"dc7f76e43c46101b47acc55ae4d593a9e6983578","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} +pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt={"checksum":"dfcb8679956b734706cf87259d50c88f83e80e66","reference":"[BACKUP-FULL-3]","size":[SIZE],"timestamp":[TIMESTAMP-1]} + +[target:file:default] +group="[GROUP-1]" +master=false +mode="0600" +user="[USER-1]" + +[target:link] +pg_data/pg_tblspc/2={"destination":"../../tablespace/ts2"} + +[target:link:default] +group="[GROUP-1]" +user="[USER-1]" + +[target:path] +pg_data={} +pg_data/base={} +pg_data/base/1={} +pg_data/base/16384={} +pg_data/base/32768={} +pg_data/global={} +pg_data/pg_clog={} +pg_data/pg_stat={} +pg_data/pg_tblspc={} +pg_tblspc={} +pg_tblspc/2={} +pg_tblspc/2/[TS_PATH-1]={} +pg_tblspc/2/[TS_PATH-1]/32768={} + +[target:path:default] +group="[GROUP-1]" +mode="0700" +user="[USER-1]" + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-3]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-5]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-3]","backup-reference":["[BACKUP-FULL-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-6]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-3]","backup-reference":["[BACKUP-FULL-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} +[BACKUP-DIFF-7]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-prior":"[BACKUP-FULL-3]","backup-reference":["[BACKUP-FULL-3]"],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"diff","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":false,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} diff --git a/test/expect/help-help-001.log b/test/expect/help-help-001.log index 2f7117a52..b289ada53 100644 --- a/test/expect/help-help-001.log +++ b/test/expect/help-help-001.log @@ -78,6 +78,13 @@ compress=n [default=3] --protocol-timeout protocol timeout [default=1830] --repo-path repository path where WAL segments and backups stored [default=/var/lib/pgbackrest] + --repo-s3-bucket s3 repository bucket + --repo-s3-endpoint s3 repository endpoint + --repo-s3-host s3 repository host + --repo-s3-key s3 repository access key + --repo-s3-key-secret s3 repository secret access key + --repo-s3-region s3 repository region + --repo-s3-verify-ssl verify S3 server certificate [default=y] --repo-type type of storage used for the repository [default=posix] --stanza defines a stanza [current=main] diff --git a/test/expect/stanza-create-001.log b/test/expect/stanza-create-001.log index 87b09a168..5a096d0c9 100644 --- a/test/expect/stanza-create-001.log +++ b/test/expect/stanza-create-001.log @@ -1,5 +1,5 @@ -run 001 - remote 0 -================== +run 001 - remote 0, s3 0 +======================== stanza-create db - fail on missing control file (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create diff --git a/test/expect/stanza-create-002.log b/test/expect/stanza-create-002.log index 921d68cf3..37854b956 100644 --- a/test/expect/stanza-create-002.log +++ b/test/expect/stanza-create-002.log @@ -1,5 +1,5 @@ -run 002 - remote 1 -================== +run 002 - remote 1, s3 0 +======================== stanza-create db - fail on missing control file (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create diff --git a/test/expect/stanza-create-003.log b/test/expect/stanza-create-003.log new file mode 100644 index 000000000..57d8f486d --- /dev/null +++ b/test/expect/stanza-create-003.log @@ -0,0 +1,456 @@ +run 003 - remote 0, s3 1 +======================== + +stanza-create db - fail on missing control file (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 ERROR: [041]: unable to open [TEST_PATH]/db-master/db/base/global/pg_control +P00 INFO: stanza-create command end: aborted with exception [041] + +stanza-create db - successfully create the stanza (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - successful rerun of stanza-create (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: archive-push command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 DEBUG: Archive::ArchivePush->process(): strWalPathFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +P00 DEBUG: Archive::ArchivePushFile::archivePushFile(): bCompress = true, strWalFile = 000000010000000100000001, strWalPath = [TEST_PATH]/db-master/db/base/pg_xlog +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp +P00 DEBUG: Archive::ArchiveCommon::walInfo(): strWalFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +P00 DEBUG: Archive::ArchiveCommon::walInfo=>: strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Archive::ArchivePushFile::archivePushCheck(): strArchiveFile = 000000010000000100000001, strDbVersion = 9.4, strWalFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, ullDbSysId = 6353949018581704918 +P00 DEBUG: Archive::ArchiveInfo->new(): bIgnoreMissing = , bLoad = , bRequired = , strArchiveClusterPath = /archive/db +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /archive/db/archive.info +P00 DEBUG: Archive::ArchiveInfo->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 6353949018581704918 +P00 DEBUG: Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef] +P00 DEBUG: Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1 +P00 DEBUG: Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1 +P00 DEBUG: Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000001 +P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000100000001, strSortOrder = +P00 DEBUG: Storage::Local->list=>: stryFileList = () +P00 DEBUG: Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = [undef] +P00 DEBUG: Archive::ArchivePushFile::archivePushCheck=>: strArchiveId = 9.4-1, strChecksum = [undef], strWarning = [undef] +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->hashSize(): xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +P00 DEBUG: Storage::Local->hashSize=>: lSize = 16777216, strHash = 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27 +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = ({strClass => pgBackRest::Storage::Filter::Gzip}), xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +P00 DEBUG: Storage::Local->openWrite(): bAtomic = true, bPathCreate = true, lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /9.4-1/000000010000000100000001-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27.gz +P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] +P00 DEBUG: Archive::ArchivePushFile::archivePushFile=>: strWarning = [undef] +P00 INFO: pushed WAL segment 000000010000000100000001 +P00 DEBUG: Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef] +P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0 +P00 DEBUG: Common::Lock::lockRelease(): bFailOnNoLock = false +P00 INFO: archive-push command end: completed successfully +P00 DEBUG: Common::Exit::exitSafe=>: iExitCode = 0 + +stanza-create db - fail on archive info file missing from non-empty dir (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 ERROR: [055]: archive information missing + HINT: use stanza-create --force to force the stanza data to be created. +P00 INFO: stanza-create command end: aborted with exception [055] + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - force create archive.info from gz file (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - repeat create (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - hash check fails requiring force (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 WARN: unable to create stanza 'db' +P00 ERROR: [028]: archive info file invalid + HINT: use stanza-upgrade if the database has been upgraded or use --force +P00 INFO: stanza-create command end: aborted with exception [028] + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="8.0" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - use force to overwrite the invalid file (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - fail on database mismatch without force option (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 WARN: unable to create stanza 'db' +P00 ERROR: [028]: archive info file invalid + HINT: use stanza-upgrade if the database has been upgraded or use --force +P00 INFO: stanza-create command end: aborted with exception [028] + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - force create archive.info from uncompressed file (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - force with missing WAL archive file (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - force with missing WAL archive directory (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} diff --git a/test/expect/stanza-upgrade-001.log b/test/expect/stanza-upgrade-001.log index 7a533655d..72832db48 100644 --- a/test/expect/stanza-upgrade-001.log +++ b/test/expect/stanza-upgrade-001.log @@ -1,5 +1,5 @@ -run 001 - local -=============== +run 001 - remote 0, s3 0 +======================== stanza-upgrade db - fail on stanza not initialized since archive.info is missing (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade diff --git a/test/expect/stanza-upgrade-002.log b/test/expect/stanza-upgrade-002.log index 347a99385..34528d795 100644 --- a/test/expect/stanza-upgrade-002.log +++ b/test/expect/stanza-upgrade-002.log @@ -1,5 +1,5 @@ -run 002 - remote -================ +run 002 - remote 1, s3 0 +======================== stanza-upgrade db - fail on stanza not initialized since archive.info is missing (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade diff --git a/test/expect/stanza-upgrade-003.log b/test/expect/stanza-upgrade-003.log new file mode 100644 index 000000000..5c9ed8539 --- /dev/null +++ b/test/expect/stanza-upgrade-003.log @@ -0,0 +1,537 @@ +run 003 - remote 1, s3 1 +======================== + +stanza-upgrade db - fail on stanza not initialized since archive.info is missing (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL segments + HINT: is archive_command configured in postgresql.conf? + HINT: has a stanza-create been performed? + HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme. +P00 INFO: stanza-upgrade command end: aborted with exception [055] + +stanza-create db - successfully create the stanza (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201306121 +db-control-version=937 +db-id=1 +db-system-id=6395542721432104958 +db-version="9.3" + +[db:history] +1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6395542721432104958 +db-version="9.3" + +[db:history] +1={"db-id":6395542721432104958,"db-version":"9.3"} + +stanza-upgrade db - already up to date (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: the stanza data is already up to date +P00 INFO: stanza-upgrade command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201306121 +db-control-version=937 +db-id=1 +db-system-id=6395542721432104958 +db-version="9.3" + +[db:history] +1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6395542721432104958 +db-version="9.3" + +[db:history] +1={"db-id":6395542721432104958,"db-version":"9.3"} + +stanza-upgrade db - fail on stanza not initialized since backup.info is missing (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 ERROR: [055]: /backup/db/backup.info does not exist and is required to perform a backup. + HINT: has a stanza-create been performed? +P00 INFO: stanza-upgrade command end: aborted with exception [055] + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6395542721432104958 +db-version="9.3" + +[db:history] +1={"db-id":6395542721432104958,"db-version":"9.3"} + +stanza-create db - use force to recreate the stanza (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201306121 +db-control-version=937 +db-id=1 +db-system-id=6395542721432104958 +db-version="9.3" + +[db:history] +1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6395542721432104958 +db-version="9.3" + +[db:history] +1={"db-id":6395542721432104958,"db-version":"9.3"} + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ +P00 ERROR: [044]: raised on 'backup remote' host: WAL segment version 9.4 does not match archive version 9.3 + WAL segment system-id 6353949018581704918 does not match archive system-id 6395542721432104958 + HINT: are you archiving to the correct stanza? + +stanza-upgrade db - successful upgrade creates mismatched files (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-upgrade command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=2 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"} +2={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=2 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6395542721432104958,"db-version":"9.3"} +2={"db-id":6353949018581704918,"db-version":"9.4"} + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ + +stanza-create db - use force to recreate the stanza producing mismatched info history but same current db-id (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=2 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"} +2={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=2 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +2={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - use force to recreate the stanza producing mismatched db-id (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=2 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +2={"db-id":6353949018581704918,"db-version":"9.4"} + +full backup - create first full backup (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --retention-full=2 --no-online --log-level-console=detail --type=full --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --retention-full=2 --stanza=db --start-fast --type=full +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/global/pg_control (8KB, 100%) checksum 89373d9f2973502940de06bc5212489df3f8a912 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%) +P00 INFO: full backup size = 16MB +P00 INFO: new backup label = [BACKUP-FULL-1] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --retention-archive=2 --retention-full=2 --stanza=db +P00 INFO: full backup total < 2 - using oldest full backup for 9.4-2 archive retention +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-user=[USER-1] + +[global] +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + +stanza-upgrade db - successfully upgrade with XX.Y-Z (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db +P00 INFO: stanza-upgrade command end: completed successfully + ++ supplemental file: /backup/db/backup.info +------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[backup:current] +[BACKUP-FULL-1]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":true,"option-hardlink":false,"option-online":false} + +[db] +db-catalog-version=201510051 +db-control-version=942 +db-id=2 +db-system-id=6392579261579036436 +db-version="9.5" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} +2={"db-catalog-version":201510051,"db-control-version":942,"db-system-id":6392579261579036436,"db-version":"9.5"} + ++ supplemental file: /archive/db/archive.info +--------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=4 +db-system-id=6392579261579036436 +db-version="9.5" + +[db:history] +2={"db-id":6353949018581704918,"db-version":"9.4"} +3={"db-id":35184388866048,"db-version":"10.0"} +4={"db-id":6392579261579036436,"db-version":"9.5"} + +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 +------------------------------------------------------------------------------------------------------------------------------------ + +diff backup - diff changed to full backup (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --retention-full=2 --no-online --log-level-console=detail --type=diff --stanza=db backup +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: backup command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --retention-full=2 --stanza=db --start-fast --type=diff +P00 WARN: no prior backup exists, diff backup has been changed to full +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 15b1a1a35c26b17570aca7920980f0ad11c6d858 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/global/pg_control (8KB, 100%) checksum e28bf39d0a56bf9fabd4049b329fcae8878bfec6 +P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%) +P00 INFO: full backup size = 16MB +P00 INFO: new backup label = [BACKUP-FULL-2] +P00 INFO: backup command end: completed successfully +P00 INFO: expire command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --retention-archive=2 --retention-full=2 --stanza=db +P00 INFO: remove archive path: /archive/db/10.0-3 +P00 INFO: expire command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf +---------------------------------------------------------- +[db] +db-path=[TEST_PATH]/db-master/db/base + +[global] +backup-cmd=[BACKREST-BIN] +backup-config=[TEST_PATH]/backup/pgbackrest.conf +backup-host=backup +backup-user=[USER-2] +db-timeout=45 +lock-path=[TEST_PATH]/db-master/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/db-master/log +protocol-timeout=60 + ++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf +------------------------------------------------------- +[db] +db-cmd=[BACKREST-BIN] +db-config=[TEST_PATH]/db-master/pgbackrest.conf +db-host=db-master +db-path=[TEST_PATH]/db-master/db/base +db-user=[USER-1] + +[global] +db-timeout=45 +lock-path=[TEST_PATH]/backup/lock +log-level-console=debug +log-level-file=trace +log-level-stderr=off +log-path=[TEST_PATH]/backup/log +protocol-timeout=60 +repo-path=/ +repo-s3-bucket=pgbackrest-dev +repo-s3-endpoint=s3.amazonaws.com +repo-s3-key=accessKey1 +repo-s3-key-secret=verySecretKey1 +repo-s3-region=us-east-1 +repo-s3-verify-ssl=n +repo-type=s3 + +[global:backup] +archive-copy=y +start-fast=y + +info all stanzas - db upgraded - db-1 and db-2 listed (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --output=json info +------------------------------------------------------------------------------------------------------------------------------------ +[ + { + "archive" : { + "max" : "000000010000000100000001", + "min" : "000000010000000100000001" + }, + "backup" : [ + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 1 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-FULL-1]", + "prior" : null, + "reference" : null, + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "full" + }, + { + "archive" : { + "start" : null, + "stop" : null + }, + "backrest" : { + "format" : 5, + "version" : "[VERSION-1]" + }, + "database" : { + "id" : 2 + }, + "info" : { + "delta" : [DELTA], + "repository" : { + "delta" : [DELTA], + "size" : [SIZE] + }, + "size" : [SIZE] + }, + "label" : "[BACKUP-FULL-2]", + "prior" : null, + "reference" : null, + "timestamp" : { + "start" : [TIMESTAMP], + "stop" : [TIMESTAMP] + }, + "type" : "full" + } + ], + "db" : [ + { + "id" : "1", + "system-id" : 6353949018581704918, + "version" : "9.4" + }, + { + "id" : "2", + "system-id" : 6392579261579036436, + "version" : "9.5" + } + ], + "name" : "db", + "status" : { + "code" : 0, + "message" : "ok" + } + } +] diff --git a/test/lib/pgBackRestTest/Common/CiTest.pm b/test/lib/pgBackRestTest/Common/CiTest.pm index 182895d8b..6de461228 100644 --- a/test/lib/pgBackRestTest/Common/CiTest.pm +++ b/test/lib/pgBackRestTest/Common/CiTest.pm @@ -163,6 +163,14 @@ sub process " libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man" . " devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl python-pip\n" . " - |\n" . + " # Install & Configure AWS CLI\n" . + " pip install --upgrade --user awscli\n" . + " aws configure set region us-east-1\n" . + " aws configure set aws_access_key_id accessKey1\n" . + " aws configure set aws_secret_access_key verySecretKey1\n" . + " aws help --version\n" . + " aws configure list\n" . + " - |\n" . " # Build Devel::Cover\n" . " git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl\n" . ' cd ~/libdevel-cover-perl && git checkout debian/' . LIB_COVER_VERSION . " && debuild -i -us -uc -b\n" . diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index edc63d770..dee3cd453 100755 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -288,6 +288,109 @@ sub coverSetup return $strScript; } +#################################################################################################################################### +# S3 server setup +#################################################################################################################################### +sub s3ServerSetup +{ + my $strOS = shift; + + # Install node.js + my $strScript = + "\n\n# Install node.js\n"; + + if ($strOS eq VM_U16) + { + $strScript .= + "RUN wget -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x && \\\n" . + " bash /root/nodejs.sh && \\\n" . + " apt-get install -y nodejs"; + } + elsif ($strOS eq VM_CO7) + { + $strScript .= + "RUN wget -O /root/nodejs.sh https://rpm.nodesource.com/setup_6.x && \\\n" . + " bash /root/nodejs.sh && \\\n" . + " yum install -y nodejs"; + } + + # Install Scality S3 + $strScript .= + "\n\n# Install Scality S3\n"; + + if ($strOS eq VM_U16) + { + $strScript .= + "RUN apt-get install -y python build-essential git && \\\n"; + } + elsif ($strOS eq VM_CO7) + { + $strScript .= + "RUN yum install -y python git && \\\n"; + } + + $strScript .= + " wget -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz && \\\n" . + " mkdir /root/scalitys3 && \\\n" . + " tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz && \\\n" . + " cd /root/scalitys3 && \\\n" . + " npm install && \\\n" . + " openssl genrsa -out ca.key 2048 && \\\n" . + " openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999" . + " -subj \"/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org\" && \\\n" . + " openssl genrsa -out server.key 2048 && \\\n" . + " openssl req -new -key server.key -out server.csr" . + " -subj \"/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org\" && \\\n" . + " openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999" . + " -sha256 && \\\n" . + ' sed -i "0,/,/s//,\n \"certFilePaths\": { \"key\": \".\/server.key\", \"cert\":' . + ' \".\/server.crt\", \"ca\": \".\/ca.crt\" },/" ./config.json' . " && \\\n" . + ' sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json'; + + return $strScript; +} + + +#################################################################################################################################### +# S3 CLI setup +#################################################################################################################################### +sub s3CliSetup +{ + my $strOS = shift; + + my $strScript = ''; + + if ($strOS eq VM_U14 || $strOS eq VM_U16 || $strOS eq VM_CO6 || $strOS eq VM_CO7) + { + # Install AWS CLI + my $strAwsPath = '/home/' . TEST_USER . '/.aws'; + + $strScript = + "\n\n# Install AWS CLI\n"; + + if ($strOS eq VM_U14 || $strOS eq VM_U16) + { + $strScript .= + "RUN apt-get install -y python-pip && \\\n"; + } + elsif ($strOS eq VM_CO6 || $strOS eq VM_CO7) + { + $strScript .= + "RUN yum -y install epel-release && \\\n" . + " yum -y update && \\\n" . + " yum -y install python-pip && \\\n"; + } + + $strScript .= + " pip install --upgrade awscli && \\\n" . + ' sudo -i -u ' . TEST_USER . " aws configure set region us-east-1 && \\\n" . + ' sudo -i -u ' . TEST_USER . " aws configure set aws_access_key_id accessKey1 && \\\n" . + ' sudo -i -u ' . TEST_USER . " aws configure set aws_secret_access_key verySecretKey1"; + } + + return $strScript; +} + #################################################################################################################################### # Install Perl packages #################################################################################################################################### @@ -312,7 +415,7 @@ sub perlInstall $strImage .= ' perl-JSON-PP'; } - $strImage .= ' perl-Digest-SHA perl-DBD-Pg'; + $strImage .= ' perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL'; } elsif ($oVm->{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) { @@ -323,6 +426,8 @@ sub perlInstall { $strImage .= ' libnet-daemon-perl libplrpc-perl'; } + + $strImage .= ' libio-socket-ssl-perl libxml-libxml-perl libhtml-parser-perl'; } else { @@ -396,7 +501,7 @@ sub containerBuild if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL) { - $strScript .= "RUN yum -y install openssh-server openssh-clients\n"; + $strScript .= "RUN yum -y install openssh-server openssh-clients wget\n"; } elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) { @@ -563,6 +668,32 @@ sub containerBuild executeTest('docker rm -f test-build'); } + # S3 image + ########################################################################################################################### + $strImageParent = $oVm->{&VM_U16}{&VM_IMAGE}; + $strImage = "${strOS}-s3-server"; + + # Required packages + $strScript = + "RUN apt-get update && \\\n" . + " apt-get install -y wget"; + + # Setup S3 server + $strScript .= s3ServerSetup(VM_U16); + + # Fix root tty + $strScript .= + "\n\n# Fix root tty\n" . + "RUN echo 'tty -s && mesg n || true' > /root/.profile"; + + # Set entrypoint + $strScript .= + "\n\nENTRYPOINT npm start --prefix /root/scalitys3"; + + # Write the image + containerWrite( + $oStorageDocker, $strTempPath, $strOS, 'S3 Server', $strImageParent, $strImage, $strScript, $bVmForce, false); + # Db image ########################################################################################################################### my @stryDbBuild; @@ -640,6 +771,9 @@ sub containerBuild # Setup sudo $strScript .= "\n\n" . sudoSetup($strOS, TEST_GROUP); + # Setup s3 cli + $strScript .= s3CliSetup($strOS); + # Write the image containerWrite( $oStorageDocker, $strTempPath, $strOS, "${strTitle} Doc", $strImageParent, $strImage, $strScript, $bVmForce); @@ -705,6 +839,15 @@ sub containerBuild # Setup Devel::Cover $strScript .= coverSetup($strOS); + # Setup S3 server + if ($strOS eq VM_U16 || $strOS eq VM_CO7) + { + $strScript .= s3ServerSetup($strOS); + } + + # Setup S3 CLI + $strScript .= s3CliSetup($strOS); + # Write the image containerWrite( $oStorageDocker, $strTempPath, $strOS, 'Loop Test', $strImageParent, $strImage, $strScript, $bVmForce, true, true); diff --git a/test/lib/pgBackRestTest/Common/DefineTest.pm b/test/lib/pgBackRestTest/Common/DefineTest.pm index 86016ded0..101231107 100644 --- a/test/lib/pgBackRestTest/Common/DefineTest.pm +++ b/test/lib/pgBackRestTest/Common/DefineTest.pm @@ -16,6 +16,8 @@ use Exporter qw(import); use pgBackRest::Common::Log; use pgBackRest::Common::String; +use pgBackRestTest::Common::VmTest; + ################################################################################################################################ # Test definition constants ################################################################################################################################ @@ -47,6 +49,9 @@ use constant TESTDEF_PROCESS => 'process' # Total runs in the test use constant TESTDEF_TOTAL => 'total'; push @EXPORT, qw(TESTDEF_TOTAL); +# VMs that the test can run on +use constant TESTDEF_VM => 'vm'; + push @EXPORT, qw(TESTDEF_VM); # The test provides full coverage for the module use constant TESTDEF_COVERAGE_FULL => true; @@ -206,6 +211,27 @@ my $oTestDef = 'Storage/Posix/FileWrite' => TESTDEF_COVERAGE_PARTIAL, }, }, + { + &TESTDEF_NAME => 's3-auth', + &TESTDEF_TOTAL => 5, + + &TESTDEF_COVERAGE => + { + 'Storage/S3/Auth' => TESTDEF_COVERAGE_FULL, + }, + }, + { + &TESTDEF_NAME => 's3', + &TESTDEF_TOTAL => 7, + &TESTDEF_VM => [VM_CO7, VM_U16], + + &TESTDEF_COVERAGE => + { + 'Storage/S3/Driver' => TESTDEF_COVERAGE_PARTIAL, + 'Storage/S3/FileRead' => TESTDEF_COVERAGE_PARTIAL, + 'Storage/S3/FileWrite' => TESTDEF_COVERAGE_FULL, + }, + }, { &TESTDEF_NAME => 'local', &TESTDEF_TOTAL => 9, @@ -292,20 +318,20 @@ my $oTestDef = }, { &TESTDEF_NAME => 'push', - &TESTDEF_TOTAL => 8, + &TESTDEF_TOTAL => 9, &TESTDEF_PROCESS => true, &TESTDEF_INDIVIDUAL => true, &TESTDEF_EXPECT => true, }, { &TESTDEF_NAME => 'stop', - &TESTDEF_TOTAL => 6, + &TESTDEF_TOTAL => 7, &TESTDEF_INDIVIDUAL => true, &TESTDEF_EXPECT => true, }, { &TESTDEF_NAME => 'get', - &TESTDEF_TOTAL => 8, + &TESTDEF_TOTAL => 9, &TESTDEF_INDIVIDUAL => true, &TESTDEF_EXPECT => true, }, @@ -370,13 +396,13 @@ my $oTestDef = }, { &TESTDEF_NAME => 'create', - &TESTDEF_TOTAL => 2, + &TESTDEF_TOTAL => 3, &TESTDEF_EXPECT => true, &TESTDEF_INDIVIDUAL => true, }, { &TESTDEF_NAME => 'upgrade', - &TESTDEF_TOTAL => 2, + &TESTDEF_TOTAL => 3, &TESTDEF_EXPECT => true, &TESTDEF_INDIVIDUAL => true, }, @@ -393,11 +419,11 @@ my $oTestDef = [ { &TESTDEF_NAME => 'synthetic', - &TESTDEF_TOTAL => 8, + &TESTDEF_TOTAL => 9, }, { &TESTDEF_NAME => 'real', - &TESTDEF_TOTAL => 11, + &TESTDEF_TOTAL => 12, &TESTDEF_DB => true, } ] @@ -432,10 +458,10 @@ foreach my $hModule (@{$oTestDef->{&TESTDEF_MODULE}}) # Resolve variables that can be set in the module or the test foreach my $strVar ( - TESTDEF_CONTAINER, TESTDEF_EXPECT, TESTDEF_PROCESS, TESTDEF_DB, TESTDEF_INDIVIDUAL) + TESTDEF_CONTAINER, TESTDEF_EXPECT, TESTDEF_PROCESS, TESTDEF_DB, TESTDEF_INDIVIDUAL, TESTDEF_VM) { $hTestDefHash->{$strModule}{$strTest}{$strVar} = coalesce( - $hModuleTest->{$strVar}, coalesce($hModule->{$strVar}, false)); + $hModuleTest->{$strVar}, $hModule->{$strVar}, $strVar eq TESTDEF_VM ? undef : false); } # Set test count diff --git a/test/lib/pgBackRestTest/Common/ExecuteTest.pm b/test/lib/pgBackRestTest/Common/ExecuteTest.pm index b23543a67..52fb89a8d 100644 --- a/test/lib/pgBackRestTest/Common/ExecuteTest.pm +++ b/test/lib/pgBackRestTest/Common/ExecuteTest.pm @@ -100,7 +100,7 @@ sub begin $self->{pId} = open3(undef, $self->{hOut}, $self->{hError}, $self->{strCommand}); # Create select objects - $self->{oIo} = new pgBackRest::Common::Io::Buffered(new pgBackRest::Common::Io::Handle('exec test', $self->{hOut}), 120, 65536); + $self->{oIo} = new pgBackRest::Common::Io::Buffered(new pgBackRest::Common::Io::Handle('exec test', $self->{hOut}), 540, 65536); if (!defined($self->{hError})) { diff --git a/test/lib/pgBackRestTest/Common/FileTest.pm b/test/lib/pgBackRestTest/Common/FileTest.pm index 57d18a070..6960d82c4 100644 --- a/test/lib/pgBackRestTest/Common/FileTest.pm +++ b/test/lib/pgBackRestTest/Common/FileTest.pm @@ -28,13 +28,17 @@ use pgBackRest::Common::Wait; use pgBackRest::Config::Config; use pgBackRest::Manifest; use pgBackRest::Storage::Local; +use pgBackRest::Storage::S3::Driver; use pgBackRestTest::Common::ExecuteTest; +use pgBackRestTest::Common::HostGroupTest; use pgBackRestTest::Common::LogTest; use pgBackRestTest::Common::VmTest; +use pgBackRestTest::Env::Host::HostBaseTest; use pgBackRestTest::Env::Host::HostBackupTest; use pgBackRestTest::Env::Host::HostDbCommonTest; use pgBackRestTest::Env::Host::HostDbTest; +use pgBackRestTest::Env::Host::HostS3Test; #################################################################################################################################### # testLinkCreate @@ -160,7 +164,11 @@ sub forceStorageMode {name => 'bRecurse', optional => true, default => false}, ); - executeTest('sudo chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp)); + # Mode commands are ignored on S3 + if ($oStorage->driver()->className() ne STORAGE_S3_DRIVER) + { + executeTest('sudo chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp)); + } # Return from function and log return values if any return logDebugReturn($strOperation); @@ -189,7 +197,18 @@ sub forceStorageMove {name => 'strDestinationPathExp'}, ); - executeTest('sudo mv ' . $oStorage->pathGet($strSourcePathExp) . ' ' . $oStorage->pathGet($strDestinationPathExp)); + # If S3 then use storage commands to remove + if ($oStorage->driver()->className() eq STORAGE_S3_DRIVER) + { + hostGroupGet()->hostGet(HOST_S3)->executeS3( + 'mv --recursive s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strSourcePathExp) . + ' s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strDestinationPathExp)); + } + # Else remove using filesystem commands + else + { + executeTest('sudo mv ' . $oStorage->pathGet($strSourcePathExp) . ' ' . $oStorage->pathGet($strDestinationPathExp)); + } # Return from function and log return values if any return logDebugReturn($strOperation); @@ -220,7 +239,11 @@ sub forceStorageOwner {name => 'bRecurse', optional => true, default => false}, ); - executeTest('sudo chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp)); + # Mode commands are ignored on S3 + if ($oStorage->driver()->className() ne STORAGE_S3_DRIVER) + { + executeTest('sudo chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp)); + } # Return from function and log return values if any return logDebugReturn($strOperation); @@ -249,7 +272,16 @@ sub forceStorageRemove {name => 'bRecurse', optional => true, default => false}, ); - executeTest('sudo rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp)); + # If S3 then use storage commands to remove + if ($oStorage->driver()->className() eq STORAGE_S3_DRIVER) + { + $oStorage->remove($strPathExp, {bRecurse => $bRecurse}); + } + # Else remove using filesystem commands + else + { + executeTest('sudo rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp)); + } # Return from function and log return values if any return logDebugReturn($strOperation); diff --git a/test/lib/pgBackRestTest/Common/ListTest.pm b/test/lib/pgBackRestTest/Common/ListTest.pm index 6125b2dbe..a7b55d68a 100644 --- a/test/lib/pgBackRestTest/Common/ListTest.pm +++ b/test/lib/pgBackRestTest/Common/ListTest.pm @@ -102,6 +102,9 @@ sub testListGet my $bFirstDbVersion = true; + # Skip this test if it can't run on this VM + next if (defined($hTest->{&TESTDEF_VM}) && grep(/^$strTestOS$/i, @{$hTest->{&TESTDEF_VM}}) == 0); + for (my $iDbVersionIdx = $iDbVersionMax; $iDbVersionIdx >= $iDbVersionMin; $iDbVersionIdx--) { if ($iDbVersionIdx == -1 || $strDbVersion eq 'all' || $strDbVersion eq 'minimal' || diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm index 7389fff4d..8b49e6805 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm @@ -26,9 +26,11 @@ use pgBackRest::Config::Config; use pgBackRest::Manifest; use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Storage::Posix::Driver; +use pgBackRest::Storage::S3::Driver; use pgBackRest::Version; use pgBackRestTest::Env::Host::HostBaseTest; +use pgBackRestTest::Env::Host::HostS3Test; use pgBackRestTest::Common::ContainerTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::HostGroupTest; @@ -87,10 +89,19 @@ sub new my $self = $class->SUPER::new($strName, {strImage => $strImage, strUser => $strUser}); bless $self, $class; - # Create repo path - $self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO; + # If repo is on local filesystem then set the repo-path locally + if ($oParam->{bRepoLocal}) + { + $self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO; + } + # Else on KV store and repo will be in root + else + { + $self->{strRepoPath} = '/'; + } - if ($$oParam{strBackupDestination} eq $self->nameGet()) + # Create the repo-path if on a local filesystem + if ($$oParam{strBackupDestination} eq $self->nameGet() && $oParam->{bRepoLocal}) { storageTest()->pathCreate($self->repoPath(), {strMode => '0770'}); } @@ -939,6 +950,18 @@ sub configCreate { $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_PATH} = $self->repoPath(); + # S3 settings + if ($oParam->{bS3}) + { + $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_TYPE} = REPO_TYPE_S3; + $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_KEY} = HOST_S3_ACCESS_KEY; + $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_KEY_SECRET} = HOST_S3_ACCESS_SECRET_KEY; + $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_BUCKET} = HOST_S3_BUCKET; + $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_ENDPOINT} = HOST_S3_ENDPOINT; + $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_REGION} = HOST_S3_REGION; + $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_VERIFY_SSL} = 'n'; + } + if (defined($$oParam{bHardlink}) && $$oParam{bHardlink}) { $self->{bHardLink} = true; @@ -1157,15 +1180,21 @@ sub infoMunge } # Modify the file/directory permissions so it can be saved + if ($self->isFS()) + { executeTest("sudo rm -f ${strFileName}* && sudo chmod 770 " . dirname($strFileName)); + } # Save the munged data to the file $oMungeIni->save(); # Fix permissions + if ($self->isFS()) + { executeTest( "sudo chmod 640 ${strFileName}* && sudo chmod 750 " . dirname($strFileName) . ' && sudo chown ' . $self->userGet() . " ${strFileName}*"); + } # Clear the cache is requested if (!$bCache) @@ -1207,18 +1236,24 @@ sub infoRestore if ($bSave) { # Modify the file/directory permissions so it can be saved + if ($self->isFS()) + { executeTest("sudo rm -f ${strFileName}* && sudo chmod 770 " . dirname($strFileName)); + } # Save the munged data to the file $self->{hInfoFile}{$strFileName}->{bModified} = true; $self->{hInfoFile}{$strFileName}->save(); # Fix permissions + if ($self->isFS()) + { executeTest( "sudo chmod 640 ${strFileName} && sudo chmod 750 " . dirname($strFileName) . ' && sudo chown ' . $self->userGet() . " ${strFileName}*"); } } + } else { confess &log(ASSERT, "There is no original data cached for $strFileName. infoMunge must be called first."); @@ -1239,6 +1274,7 @@ sub backupDestination {return shift->{strBackupDestination}} sub backrestExe {return testRunGet()->backrestExe()} sub hardLink {return shift->{bHardLink}} sub hasLink {storageRepo()->driver()->className() eq STORAGE_POSIX_DRIVER} +sub isFS {storageRepo()->driver()->className() ne STORAGE_S3_DRIVER} sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()} sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER} sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY} diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm index 8a1d0d2b1..bedcd8a40 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm @@ -74,6 +74,7 @@ sub new strBackupDestination => $$oParam{strBackupDestination}, oLogTest => $$oParam{oLogTest}, bSynthetic => $$oParam{bSynthetic}, + bRepoLocal => $oParam->{bRepoLocal}, }); bless $self, $class; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm index 3c54910d8..aaa0797c4 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm @@ -60,6 +60,7 @@ sub new oLogTest => $$oParam{oLogTest}, bSynthetic => true, bStandby => $$oParam{bStandby}, + bRepoLocal => $oParam->{bRepoLocal}, }); bless $self, $class; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm index 91efee6ba..5ff849d7e 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm @@ -74,6 +74,7 @@ sub new strBackupDestination => $$oParam{strBackupDestination}, oLogTest => $$oParam{oLogTest}, bStandby => $$oParam{bStandby}, + bRepoLocal => $oParam->{bRepoLocal}, }); bless $self, $class; diff --git a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm new file mode 100644 index 000000000..d3928067c --- /dev/null +++ b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm @@ -0,0 +1,106 @@ +#################################################################################################################################### +# S3 Test Host +#################################################################################################################################### +package pgBackRestTest::Env::Host::HostS3Test; +use parent 'pgBackRestTest::Common::HostTest'; + +#################################################################################################################################### +# Perl includes +#################################################################################################################################### +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); + +use Exporter qw(import); + our @EXPORT = qw(); +use File::Basename qw(dirname); +use Storable qw(dclone); + +use pgBackRest::Backup::Common; +use pgBackRest::Common::Exception; +use pgBackRest::Common::Ini; +use pgBackRest::Common::Log; +use pgBackRest::Config::Config; +use pgBackRest::Manifest; +use pgBackRest::Protocol::Storage::Helper; +use pgBackRest::Version; + +use pgBackRestTest::Env::Host::HostBaseTest; +use pgBackRestTest::Common::ContainerTest; +use pgBackRestTest::Common::ExecuteTest; +use pgBackRestTest::Common::HostGroupTest; +use pgBackRestTest::Common::RunTest; + +#################################################################################################################################### +# S3 defaults +#################################################################################################################################### +use constant HOST_S3_ACCESS_KEY => 'accessKey1'; + push @EXPORT, qw(HOST_S3_ACCESS_KEY); +use constant HOST_S3_ACCESS_SECRET_KEY => 'verySecretKey1'; + push @EXPORT, qw(HOST_S3_ACCESS_SECRET_KEY); +use constant HOST_S3_BUCKET => 'pgbackrest-dev'; + push @EXPORT, qw(HOST_S3_BUCKET); +use constant HOST_S3_ENDPOINT => 's3.amazonaws.com'; + push @EXPORT, qw(HOST_S3_ENDPOINT); +use constant HOST_S3_REGION => 'us-east-1'; + push @EXPORT, qw(HOST_S3_REGION); + +#################################################################################################################################### +# new +#################################################################################################################################### +sub new +{ + my $class = shift; # Class name + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + ) = + logDebugParam + ( + __PACKAGE__ . '->new', \@_, + ); + + # Create the host + my $self = $class->SUPER::new( + HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', containerRepo() . ':' . testRunGet()->vm() . '-s3-server', + 'root', testRunGet()->vm()); + bless $self, $class; + + # Return from function and log return values if any + return logDebugReturn + ( + $strOperation, + {name => 'self', value => $self, trace => true} + ); +} + +#################################################################################################################################### +# executeS3 +#################################################################################################################################### +sub executeS3 +{ + my $self = shift; + + # Assign function parameters, defaults, and log debug info + my + ( + $strOperation, + $strCommand + ) = + logDebugParam + ( + __PACKAGE__ . '->executeS3', \@_, + {name => 'strCommand', trace => true}, + ); + + executeTest( + 'export PYTHONWARNINGS="ignore" && aws --endpoint-url=https://' . $self->ipGet() . + ' s3 --no-verify-ssl ' . $strCommand); + + # Return from function and log return values if any + return logDebugReturn($strOperation); +} + +1; diff --git a/test/lib/pgBackRestTest/Env/HostEnvTest.pm b/test/lib/pgBackRestTest/Env/HostEnvTest.pm index 0745c6983..5ae439334 100644 --- a/test/lib/pgBackRestTest/Env/HostEnvTest.pm +++ b/test/lib/pgBackRestTest/Env/HostEnvTest.pm @@ -24,6 +24,9 @@ use pgBackRestTest::Env::Host::HostBaseTest; use pgBackRestTest::Env::Host::HostDbCommonTest; use pgBackRestTest::Env::Host::HostDbTest; use pgBackRestTest::Env::Host::HostDbSyntheticTest; +use pgBackRestTest::Env::Host::HostS3Test; +use pgBackRestTest::Common::ContainerTest; +use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::HostGroupTest; use pgBackRestTest::Common::RunTest; @@ -57,6 +60,15 @@ sub setup my $oLogTest = shift; my $oConfigParam = shift; + # Start S3 server first since it takes the longest + #------------------------------------------------------------------------------------------------------------------------------- + my $oHostS3; + + if ($oConfigParam->{bS3}) + { + $oHostS3 = new pgBackRestTest::Env::Host::HostS3Test(); + } + # Get host group my $oHostGroup = hostGroupGet(); @@ -70,7 +82,8 @@ sub setup $strBackupDestination = defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_BACKUP; $oHostBackup = new pgBackRestTest::Env::Host::HostBackupTest( - {strBackupDestination => $strBackupDestination, bSynthetic => $bSynthetic, oLogTest => $oLogTest}); + {strBackupDestination => $strBackupDestination, bSynthetic => $bSynthetic, oLogTest => $oLogTest, + bRepoLocal => !$oConfigParam->{bS3}}); $oHostGroup->hostAdd($oHostBackup); } else @@ -85,12 +98,12 @@ sub setup if ($bSynthetic) { $oHostDbMaster = new pgBackRestTest::Env::Host::HostDbSyntheticTest( - {strBackupDestination => $strBackupDestination, oLogTest => $oLogTest}); + {strBackupDestination => $strBackupDestination, oLogTest => $oLogTest, bRepoLocal => !$oConfigParam->{bS3}}); } else { $oHostDbMaster = new pgBackRestTest::Env::Host::HostDbTest( - {strBackupDestination => $strBackupDestination, oLogTest => $oLogTest}); + {strBackupDestination => $strBackupDestination, oLogTest => $oLogTest, bRepoLocal => !$oConfigParam->{bS3}}); } $oHostGroup->hostAdd($oHostDbMaster); @@ -101,24 +114,38 @@ sub setup if (defined($$oConfigParam{bStandby}) && $$oConfigParam{bStandby}) { $oHostDbStandby = new pgBackRestTest::Env::Host::HostDbTest( - {strBackupDestination => $strBackupDestination, bStandby => true, oLogTest => $oLogTest}); + {strBackupDestination => $strBackupDestination, bStandby => true, oLogTest => $oLogTest, + bRepoLocal => !$oConfigParam->{bS3}}); $oHostGroup->hostAdd($oHostDbStandby); } + # Finalize S3 server + #------------------------------------------------------------------------------------------------------------------------------- + if (defined($oHostS3)) + { + $oHostGroup->hostAdd($oHostS3, {rstryHostName => ['pgbackrest-dev.s3.amazonaws.com', 's3.amazonaws.com']}); + + # Wait for server to start + executeTest('docker logs -f ' . $oHostS3->container() . " | grep -m 1 \"server started\""); + + $oHostS3->executeS3('mb s3://' . HOST_S3_BUCKET); + } # Create db master config $oHostDbMaster->configCreate({ strBackupSource => $$oConfigParam{strBackupSource}, bCompress => $$oConfigParam{bCompress}, bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink}, - bArchiveAsync => $$oConfigParam{bArchiveAsync}}); + bArchiveAsync => $$oConfigParam{bArchiveAsync}, + bS3 => $$oConfigParam{bS3}}); # Create backup config if backup host exists if (defined($oHostBackup)) { $oHostBackup->configCreate({ bCompress => $$oConfigParam{bCompress}, - bHardlink => $$oConfigParam{bHardLink}}); + bHardlink => $$oConfigParam{bHardLink}, + bS3 => $$oConfigParam{bS3}}); } # If backup host is not defined set it to db-master else @@ -141,9 +168,23 @@ sub setup $self->optionSetTest($oOption, OPTION_DB_PATH, $oHostDbMaster->dbBasePath()); $self->optionSetTest($oOption, OPTION_REPO_PATH, $oHostBackup->repoPath()); $self->optionSetTest($oOption, OPTION_STANZA, $self->stanza()); + + # Set S3 options + if (defined($oHostS3)) + { + $self->optionSetTest($oOption, OPTION_REPO_TYPE, REPO_TYPE_S3); + $self->optionSetTest($oOption, OPTION_REPO_S3_KEY, HOST_S3_ACCESS_KEY); + $self->optionSetTest($oOption, OPTION_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY); + $self->optionSetTest($oOption, OPTION_REPO_S3_BUCKET, HOST_S3_BUCKET); + $self->optionSetTest($oOption, OPTION_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT); + $self->optionSetTest($oOption, OPTION_REPO_S3_REGION, HOST_S3_REGION); + $self->optionSetTest($oOption, OPTION_REPO_S3_HOST, $oHostS3->ipGet()); + $self->optionBoolSetTest($oOption, OPTION_REPO_S3_VERIFY_SSL, false); + } + $self->testResult(sub {$self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH)}, '', 'config load'); - return $oHostDbMaster, $oHostDbStandby, $oHostBackup; + return $oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3; } #################################################################################################################################### diff --git a/test/lib/pgBackRestTest/Env/S3EnvTest.pm b/test/lib/pgBackRestTest/Env/S3EnvTest.pm new file mode 100644 index 000000000..430d74b94 --- /dev/null +++ b/test/lib/pgBackRestTest/Env/S3EnvTest.pm @@ -0,0 +1,56 @@ +#################################################################################################################################### +# S3 Test Environment +#################################################################################################################################### +package pgBackRestTest::Env::S3EnvTest; +use parent 'pgBackRestTest::Common::RunTest'; + +#################################################################################################################################### +# Perl includes +#################################################################################################################################### +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use pgBackRest::Common::Log; +use pgBackRest::Common::String; +use pgBackRest::Storage::S3::Driver; + +use pgBackRestTest::Common::ExecuteTest; +use pgBackRestTest::Common::RunTest; + +#################################################################################################################################### +# initS3 +#################################################################################################################################### +sub initS3 +{ + my $self = shift; + + my ($strBucket, $strEndPoint, $strRegion, $strAccessKeyId, $strSecretAccessKey) = + ('pgbackrest-dev', 's3.amazonaws.com', 'us-east-1', 'accessKey1', 'verySecretKey1'); + + my $strS3ServerPath = $self->testPath() . '/s3server'; + my $strS3ServerDataPath = "${strS3ServerPath}/data"; + my $strS3ServerMetaPath = "${strS3ServerPath}/meta"; + my $strS3ServerLogFile = "${strS3ServerPath}/server.log"; + storageTest()->pathCreate($strS3ServerDataPath, {bCreateParent => true}); + storageTest()->pathCreate($strS3ServerMetaPath, {bCreateParent => true}); + + $self->{strS3Command} = 'export PYTHONWARNINGS="ignore" && aws s3 --no-verify-ssl'; + + executeTest("echo '127.0.0.1 ${strBucket}.${strEndPoint} ${strEndPoint}' | sudo tee -a /etc/hosts"); + executeTest('sudo sed -i "s/logLevel\"\: \"info\"/logLevel\"\: \"trace\"/" /root/scalitys3/config.json'); + executeTest("sudo npm start --prefix /root/scalitys3 > ${strS3ServerLogFile} 2>&1 &"); + executeTest("tail -f ${strS3ServerLogFile} | grep -m 1 \"server started\""); + executeTest("$self->{strS3Command} mb s3://pgbackrest-dev"); + + # Test variables + my $strFile = 'file.txt'; + my $strFileContent = 'TESTDATA'; + + # Initialize the driver + return new pgBackRest::Storage::S3::Driver( + $strBucket, $strEndPoint, $strRegion, $strAccessKeyId, $strSecretAccessKey, {bVerifySsl => false, lBufferMax => 1048576}); +} + +1; diff --git a/test/lib/pgBackRestTest/Module/Archive/ArchiveGetTest.pm b/test/lib/pgBackRestTest/Module/Archive/ArchiveGetTest.pm index 1f88e2d43..5e2410701 100644 --- a/test/lib/pgBackRestTest/Module/Archive/ArchiveGetTest.pm +++ b/test/lib/pgBackRestTest/Module/Archive/ArchiveGetTest.pm @@ -42,18 +42,20 @@ sub run my $iArchiveMax = 3; my $strArchiveTestFile = $self->dataPath() . '/backup.wal2_' . WAL_VERSION_94 . '.bin'; - for (my $bRemote = false; $bRemote <= true; $bRemote++) + foreach my $bS3 (false, true) { - for (my $bCompress = false; $bCompress <= true; $bCompress++) + foreach my $bRemote ($bS3 ? (true) : (false, true)) { - for (my $bExists = false; $bExists <= true; $bExists++) + foreach my $bCompress ($bS3 ? (true) : (false, true)) + { + foreach my $bExists ($bS3 ? (true) : (false, true)) { # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, exists ${bExists}")) {next} + if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, exists ${bExists}, s3 ${bS3}")) {next} # Create hosts and config - my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress}); + my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( + true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bS3 => $bS3}); # Create the xlog path my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog'; @@ -175,6 +177,7 @@ sub run } } } + } } 1; diff --git a/test/lib/pgBackRestTest/Module/Archive/ArchivePushTest.pm b/test/lib/pgBackRestTest/Module/Archive/ArchivePushTest.pm index 5d54f2ff0..069cccdc7 100644 --- a/test/lib/pgBackRestTest/Module/Archive/ArchivePushTest.pm +++ b/test/lib/pgBackRestTest/Module/Archive/ArchivePushTest.pm @@ -80,18 +80,22 @@ sub run my $strArchiveChecksum = '72b9da071c13957fb4ca31f05dbd5c644297c2f7'; my $iArchiveMax = 3; - for (my $bRemote = false; $bRemote <= true; $bRemote++) + foreach my $bS3 (false, true) { - for (my $bCompress = false; $bCompress <= true; $bCompress++) + foreach my $bRemote ($bS3 ? (true) : (false, true)) { - for (my $bArchiveAsync = false; $bArchiveAsync <= true; $bArchiveAsync++) + foreach my $bCompress ($bS3 ? (true) : (false, true)) + { + foreach my $bArchiveAsync ($bS3 ? (true) : (false, true)) { # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, arc_async ${bArchiveAsync}", $self->processMax() == 1)) {next} + if (!$self->begin( + "rmt ${bRemote}, cmp ${bCompress}, arc_async ${bArchiveAsync}, s3 ${bS3}", $self->processMax() == 1)) {next} # Create hosts, file object, and config my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => $bArchiveAsync}); + true, $self->expect(), + {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => $bArchiveAsync, bS3 => $bS3}); # Create the xlog path my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog'; @@ -151,10 +155,10 @@ sub run ', archive ' .sprintf('%02x', $iArchive) . " - ${strArchiveFile}"); - # Create a temp file to make sure it is deleted later + # Create a temp file to make sure it is deleted later (skip when S3 since it doesn't use temp files) my $strArchiveTmp; - if ($iBackup == 1 && $iArchive == 2) + if (!$bS3 && $iBackup == 1 && $iArchive == 2) { # Should succeed when temp file already exists &log(INFO, ' test archive when tmp file exists'); @@ -344,6 +348,7 @@ sub run } } } + } } 1; diff --git a/test/lib/pgBackRestTest/Module/Archive/ArchiveStopTest.pm b/test/lib/pgBackRestTest/Module/Archive/ArchiveStopTest.pm index d07c3a813..e1aa7da24 100644 --- a/test/lib/pgBackRestTest/Module/Archive/ArchiveStopTest.pm +++ b/test/lib/pgBackRestTest/Module/Archive/ArchiveStopTest.pm @@ -38,18 +38,20 @@ sub run my $strArchiveTestFile = $self->dataPath() . '/backup.wal2_' . WAL_VERSION_94 . '.bin'; - for (my $bRemote = false; $bRemote <= true; $bRemote++) + foreach my $bS3 (false, true) { - for (my $bCompress = false; $bCompress <= true; $bCompress++) + foreach my $bRemote ($bS3 ? (true) : (false, true)) { - for (my $iError = 0; $iError <= $bRemote; $iError++) + foreach my $bCompress ($bS3 ? (false) : (false, true)) + { + foreach my $iError ($bS3 ? (1) : ($bRemote ? (0, 1) : (0))) { # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, error " . ($iError ? 'connect' : 'version'))) {next} + if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, error " . ($iError ? 'connect' : 'version') . ", s3 ${bS3}")) {next} # Create hosts, file object, and config - my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true}); + my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( + true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true, bS3 => $bS3}); my $oStorage = storageRepo(); @@ -117,6 +119,7 @@ sub run } } } + } } 1; diff --git a/test/lib/pgBackRestTest/Module/Expire/ExpireExpireTest.pm b/test/lib/pgBackRestTest/Module/Expire/ExpireExpireTest.pm index f02b33f62..c5ad37565 100644 --- a/test/lib/pgBackRestTest/Module/Expire/ExpireExpireTest.pm +++ b/test/lib/pgBackRestTest/Module/Expire/ExpireExpireTest.pm @@ -29,6 +29,7 @@ use pgBackRest::Protocol::Storage::Helper; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::RunTest; use pgBackRestTest::Env::ExpireEnvTest; +use pgBackRestTest::Env::Host::HostS3Test; use pgBackRestTest::Env::HostEnvTest; #################################################################################################################################### @@ -40,6 +41,7 @@ sub initStanzaOption my $oOption = shift; my $strDbBasePath = shift; my $strRepoPath = shift; + my $oHostS3 = shift; $self->optionSetTest($oOption, OPTION_STANZA, $self->stanza()); $self->optionSetTest($oOption, OPTION_DB_PATH, $strDbBasePath); @@ -50,6 +52,18 @@ sub initStanzaOption $self->optionSetTest($oOption, OPTION_DB_TIMEOUT, 5); $self->optionSetTest($oOption, OPTION_PROTOCOL_TIMEOUT, 6); + + if (defined($oHostS3)) + { + $self->optionSetTest($oOption, OPTION_REPO_TYPE, REPO_TYPE_S3); + $self->optionSetTest($oOption, OPTION_REPO_S3_KEY, HOST_S3_ACCESS_KEY); + $self->optionSetTest($oOption, OPTION_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY); + $self->optionSetTest($oOption, OPTION_REPO_S3_BUCKET, HOST_S3_BUCKET); + $self->optionSetTest($oOption, OPTION_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT); + $self->optionSetTest($oOption, OPTION_REPO_S3_REGION, HOST_S3_REGION); + $self->optionSetTest($oOption, OPTION_REPO_S3_HOST, $oHostS3->ipGet()); + $self->optionBoolSetTest($oOption, OPTION_REPO_S3_VERIFY_SSL, false); + } } #################################################################################################################################### @@ -64,12 +78,14 @@ sub run my $strDescription; my $oOption = {}; + my $bS3 = false; + if ($self->begin("local")) { # Create hosts, file object, and config - my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(true, $self->expect()); + my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(true, $self->expect(), {bS3 => $bS3}); - $self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}); + $self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); # Create the test object @@ -164,9 +180,9 @@ sub run if ($self->begin("Expire::stanzaUpgrade")) { # Create hosts, file object, and config - my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(true, $self->expect()); + my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(true, $self->expect(), {bS3 => $bS3}); - $self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}); + $self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3); $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE); # Create the test object diff --git a/test/lib/pgBackRestTest/Module/Full/FullRealTest.pm b/test/lib/pgBackRestTest/Module/Full/FullRealTest.pm index 9aac57b86..3caaa1017 100644 --- a/test/lib/pgBackRestTest/Module/Full/FullRealTest.pm +++ b/test/lib/pgBackRestTest/Module/Full/FullRealTest.pm @@ -43,20 +43,23 @@ sub run { my $self = shift; - foreach my $bHostBackup (false, true) + foreach my $bS3 (false, true) { - foreach my $bHostStandby (false, true) + foreach my $bHostBackup ($bS3 ? (true) : (false, true)) + { + foreach my $bHostStandby ($bS3 ? (false) : (false, true)) { foreach my $strBackupDestination ( - $bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER)) + $bS3 || $bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER)) { - foreach my $bArchiveAsync ($bHostStandby ? (false) : (false, true)) + foreach my $bArchiveAsync ($bS3 ? (true) : ($bHostStandby ? (false) : (false, true))) { - foreach my $bCompress ($bHostStandby ? (false) : (false, true)) + foreach my $bCompress ($bS3 ? (true) : ($bHostStandby ? (false) : (false, true))) { # Increment the run, log, and decide whether this unit test should be run next if (!$self->begin( - "bkp ${bHostBackup}, sby ${bHostStandby}, dst ${strBackupDestination}, asy ${bArchiveAsync}, cmp ${bCompress}", + "bkp ${bHostBackup}, sby ${bHostStandby}, dst ${strBackupDestination}, asy ${bArchiveAsync}, cmp ${bCompress}," . + " s3 ${bS3}", $self->processMax() == 1 && $self->pgVersion() eq PG_VERSION_95)); if ($bHostStandby && $self->pgVersion() < PG_VERSION_HOT_STANDBY) @@ -66,13 +69,13 @@ sub run } # Create hosts, file object, and config - my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup( + my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( false, $self->expect(), {bHostBackup => $bHostBackup, bStandby => $bHostStandby, strBackupDestination => $strBackupDestination, - bCompress => $bCompress, bArchiveAsync => $bArchiveAsync}); + bCompress => $bCompress, bArchiveAsync => $bArchiveAsync, bS3 => $bS3}); # Determine if extra tests are performed. Extra tests should not be primary tests for compression or async archiving. - my $bTestExtra = ($self->runCurrent() == 1) || ($self->runCurrent() == 7); + my $bTestExtra = $self->runCurrent() == 1 || $self->runCurrent() == 7 || $self->runCurrent() == 12; $oHostDbMaster->clusterCreate(); @@ -251,8 +254,11 @@ sub run forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE); forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT); - $oHostBackup->stanzaCreate('fail on backup info file missing from non-empty dir', - {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY}); + if (!$bS3) + { + $oHostBackup->stanzaCreate('fail on backup info file missing from non-empty dir', + {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY}); + } # Force the backup.info file to be recreated $oHostBackup->stanzaCreate('verify success with force', {strOptionalParam => ' --' . OPTION_FORCE}); @@ -899,6 +905,7 @@ sub run } } } + } } 1; diff --git a/test/lib/pgBackRestTest/Module/Full/FullSyntheticTest.pm b/test/lib/pgBackRestTest/Module/Full/FullSyntheticTest.pm index 3d2af97fa..56ebfa6fe 100644 --- a/test/lib/pgBackRestTest/Module/Full/FullSyntheticTest.pm +++ b/test/lib/pgBackRestTest/Module/Full/FullSyntheticTest.pm @@ -29,11 +29,12 @@ use pgBackRest::Protocol::Storage::Helper; use pgBackRest::Version; use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Env::HostEnvTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::FileTest; use pgBackRestTest::Common::RunTest; use pgBackRestTest::Env::Host::HostBackupTest; +use pgBackRestTest::Env::Host::HostS3Test; +use pgBackRestTest::Env::HostEnvTest; #################################################################################################################################### # Build PostgreSQL pages for testing @@ -58,18 +59,20 @@ sub run { my $self = shift; - for (my $bRemote = false; $bRemote <= true; $bRemote++) + foreach my $bS3 (false, true) { - for (my $bCompress = false; $bCompress <= true; $bCompress++) + foreach my $bRemote ($bS3 ? (true) : (false, true)) { - for (my $bHardLink = false; $bHardLink <= true; $bHardLink++) + foreach my $bCompress ($bS3 ? (false) : (false, true)) + { + foreach my $bHardLink ($bS3 ? (false) : (false, true)) { # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, hardlink ${bHardLink}", $self->processMax() == 1)) {next} + if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, hardlink ${bHardLink}, s3 ${bS3}", $self->processMax() == 1)) {next} # Create hosts, file object, and config - my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bHardLink => $bHardLink}); + my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( + true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bHardLink => $bHardLink, bS3 => $bS3}); # Determine if this is a neutral test, i.e. we only want to do it once for local and once for remote. Neutral means # that options such as compression and hardlinks are disabled @@ -316,7 +319,7 @@ sub run {oExpectedManifest => \%oManifest, strOptionalParam => $strOptionalParam . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . ' --' . OPTION_BUFFER_SIZE . '=16384 --' . OPTION_CHECKSUM_PAGE, - strRepoType => REPO_TYPE_CIFS, strTest => $strTestPoint, fTestDelay => 0}); + strRepoType => $bS3 ? undef : REPO_TYPE_CIFS, strTest => $strTestPoint, fTestDelay => 0}); # Error on backup option to check logging #----------------------------------------------------------------------------------------------------------------------- @@ -435,9 +438,17 @@ sub run forceStorageRemove(storageRepo(), "${strResumePath}/" . FILE_MANIFEST); # Create a temp file in backup temp root to be sure it's deleted correctly - executeTest("sudo touch ${strResumePath}/file.tmp" . ($bCompress ? '.gz' : ''), - {bRemote => $bRemote}); - executeTest("sudo chown " . BACKREST_USER . " ${strResumePath}/file.tmp" . ($bCompress ? '.gz' : '')); + my $strTempFile = "${strResumePath}/file.tmp" . ($bCompress ? '.gz' : ''); + + if ($bS3) + { + $oHostS3->executeS3('cp /etc/hosts s3://' . HOST_S3_BUCKET . "${strTempFile}"); + } + else + { + executeTest("sudo touch ${strTempFile}", {bRemote => $bRemote}); + executeTest("sudo chown " . BACKREST_USER . " ${strResumePath}/file.tmp" . ($bCompress ? '.gz' : '')); + } $strFullBackup = $oHostBackup->backup( $strType, 'resume', @@ -454,7 +465,7 @@ sub run $oHostBackup->backup( $strType, 'invalid repo', {oExpectedManifest => \%oManifest, strOptionalParam => '--' . OPTION_REPO_PATH . '=/bogus_path' . - ' --log-level-console=detail', iExpectedExitStatus => ERROR_PATH_MISSING}); + ' --log-level-console=detail', iExpectedExitStatus => $bS3 ? ERROR_FILE_MISSING : ERROR_PATH_MISSING}); } # Restore - tests various mode, extra files/paths, missing files/paths @@ -1185,6 +1196,7 @@ sub run } } } + } } 1; diff --git a/test/lib/pgBackRestTest/Module/Stanza/StanzaCreateTest.pm b/test/lib/pgBackRestTest/Module/Stanza/StanzaCreateTest.pm index f6f4b0284..631d6d725 100644 --- a/test/lib/pgBackRestTest/Module/Stanza/StanzaCreateTest.pm +++ b/test/lib/pgBackRestTest/Module/Stanza/StanzaCreateTest.pm @@ -40,14 +40,16 @@ sub run { my $self = shift; - foreach my $bRemote (false, true) + foreach my $bS3 (false, true) + { + foreach my $bRemote ($bS3 ? (false) : (false, true)) { # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin("remote ${bRemote}")) {next} + if (!$self->begin("remote $bRemote, s3 $bS3")) {next} # Create hosts, file object, and config - my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote}); + my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( + true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3}); # Create the stanza $oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_OPEN, @@ -83,21 +85,25 @@ sub run $oHostBackup->stanzaCreate('fail on archive info file missing from non-empty dir', {iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . OPTION_ONLINE}); - # Change the permissions of the archive file so it cannot be read - forceStorageMode( - storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' . - COMPRESS_EXT, - '220'); + # S3 doesn't support filesystem-style permissions so skip these tests + if (!$bS3) + { + # Change the permissions of the archive file so it cannot be read + forceStorageMode( + storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' . + COMPRESS_EXT, + '220'); - # Force creation of the info file but fail on gunzip - $oHostBackup->stanzaCreate('gunzip fail on forced stanza-create', - {iExpectedExitStatus => ERROR_FILE_OPEN, strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE}); + # Force creation of the info file but fail on gunzip + $oHostBackup->stanzaCreate('gunzip fail on forced stanza-create', + {iExpectedExitStatus => ERROR_FILE_OPEN, strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE}); - # Change permissions back - forceStorageMode( - storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' . - COMPRESS_EXT, - '640'); + # Change permissions back + forceStorageMode( + storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' . + COMPRESS_EXT, + '640'); + } # Force creation of archive info from the gz file $oHostBackup->stanzaCreate('force create archive.info from gz file', @@ -170,6 +176,7 @@ sub run $oHostBackup->stanzaCreate('force with missing WAL archive directory', {strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE}); } + } } 1; diff --git a/test/lib/pgBackRestTest/Module/Stanza/StanzaUpgradeTest.pm b/test/lib/pgBackRestTest/Module/Stanza/StanzaUpgradeTest.pm index 9f79cbb32..85f5d6e39 100644 --- a/test/lib/pgBackRestTest/Module/Stanza/StanzaUpgradeTest.pm +++ b/test/lib/pgBackRestTest/Module/Stanza/StanzaUpgradeTest.pm @@ -43,14 +43,16 @@ sub run { my $self = shift; - for (my $bRemote = false; $bRemote <= true; $bRemote++) + foreach my $bS3 (false, true) + { + foreach my $bRemote ($bS3 ? (true) : (false, true)) { # Increment the run, log, and decide whether this unit test should be run - if (!$self->begin($bRemote ? "remote" : "local")) {next} + if (!$self->begin("remote $bRemote, s3 $bS3")) {next} # Create hosts, file object, and config - my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup( - true, $self->expect(), {bHostBackup => $bRemote}); + my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup( + true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3}); # Create the test path for pg_control storageDb()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true}); @@ -169,6 +171,7 @@ sub run # Confirm info command displays the JSON correctly $oHostDbMaster->info('db upgraded - db-1 and db-2 listed', {strOutput => INFO_OUTPUT_JSON}); } + } } 1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageS3AuthTest.pm b/test/lib/pgBackRestTest/Module/Storage/StorageS3AuthTest.pm new file mode 100644 index 000000000..0e0e07d42 --- /dev/null +++ b/test/lib/pgBackRestTest/Module/Storage/StorageS3AuthTest.pm @@ -0,0 +1,111 @@ +#################################################################################################################################### +# S3 Authentication Tests +#################################################################################################################################### +package pgBackRestTest::Module::Storage::StorageS3AuthTest; +use parent 'pgBackRestTest::Common::RunTest'; + +#################################################################################################################################### +# Perl includes +#################################################################################################################################### +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use POSIX qw(strftime); + +use pgBackRest::Common::Exception; +use pgBackRest::Common::Log; +use pgBackRest::Common::Wait; +use pgBackRest::Storage::S3::Auth; + +#################################################################################################################################### +# run +#################################################################################################################################### +sub run +{ + my $self = shift; + + ################################################################################################################################ + if ($self->begin('s3DateTime')) + { + $self->testResult(sub {s3DateTime(1492127085)}, '20170413T234445Z', 'format date/time'); + + #--------------------------------------------------------------------------------------------------------------------------- + waitRemainder(); + $self->testResult(sub {s3DateTime()}, strftime("%Y%m%dT%k%M%SZ", gmtime()), 'format current date/time'); + } + + ################################################################################################################################ + if ($self->begin('s3CanonicalRequest')) + { + $self->testResult( + sub {s3CanonicalRequest( + 'GET', qw(/), 'list-type=2', + {'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z', + 'x-amz-content-sha256' => '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9'}, + '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')}, + "(GET\n/\nlist-type=2\nhost:bucket.s3.amazonaws.com\n" . + "x-amz-content-sha256:705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9\n" . + "x-amz-date:20170606T121212Z\n\nhost;x-amz-content-sha256;x-amz-date\n" . + '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9' . + ', host;x-amz-content-sha256;x-amz-date)', + 'canonical request'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testException( + sub {s3CanonicalRequest( + 'GET', qw(/), 'list-type=2', {'Host' => 'bucket.s3.amazonaws.com'}, + '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')}, + ERROR_ASSERT, "header 'Host' must be lower case"); + } + + ################################################################################################################################ + if ($self->begin('s3SigningKey')) + { + $self->testResult( + sub {unpack('H*', s3SigningKey('20170412', 'us-east-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))}, + '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9', 'signing key'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {unpack('H*', s3SigningKey('20170412', 'us-east-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))}, + '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9', 'same signing key from cache'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {unpack('H*', s3SigningKey('20170505', 'us-west-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))}, + 'c1a1cb590bbc38ba789c8e5695a1ec0cd7fd44c6949f922e149005a221524c09', 'new signing key'); + } + + ################################################################################################################################ + if ($self->begin('s3StringToSign')) + { + $self->testResult( + sub {s3StringToSign( + '20170412T141414Z', 'us-east-1', '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')}, + "AWS4-HMAC-SHA256\n20170412T141414Z\n20170412/us-east-1/s3/aws4_request\n" . + "705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9", + 'string to sign'); + } + + ################################################################################################################################ + if ($self->begin('s3AuthorizationHeader')) + { + $self->testResult( + sub {s3AuthorizationHeader( + 'us-east-1', 'bucket.s3.amazonaws.com', 'GET', qw(/), 'list-type=2', '20170606T121212Z', + {'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z'}, + 'AKIAIOSFODNN7EXAMPLE', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')}, + '{authorization => AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,' . + 'SignedHeaders=host;x-amz-content-sha256;x-amz-date,' . + 'Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3,' . + ' host => bucket.s3.amazonaws.com,' . + ' x-amz-content-sha256 => e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,' . + ' x-amz-date => 20170606T121212Z}', + 'authorization header request'); + } +} + +1; diff --git a/test/lib/pgBackRestTest/Module/Storage/StorageS3Test.pm b/test/lib/pgBackRestTest/Module/Storage/StorageS3Test.pm new file mode 100644 index 000000000..bdb3ffdff --- /dev/null +++ b/test/lib/pgBackRestTest/Module/Storage/StorageS3Test.pm @@ -0,0 +1,208 @@ +#################################################################################################################################### +# S3 Storage Tests +#################################################################################################################################### +package pgBackRestTest::Module::Storage::StorageS3Test; +use parent 'pgBackRestTest::Env::S3EnvTest'; + +#################################################################################################################################### +# Perl includes +#################################################################################################################################### +use strict; +use warnings FATAL => qw(all); +use Carp qw(confess); +use English '-no_match_vars'; + +use Digest::SHA qw(sha1_hex); + +use pgBackRest::Common::Log; +use pgBackRest::Common::String; +use pgBackRest::Storage::S3::Driver; + +use pgBackRestTest::Common::ExecuteTest; +use pgBackRestTest::Common::RunTest; + +#################################################################################################################################### +# initTest +#################################################################################################################################### +sub initTest +{ + my $self = shift; + + executeTest("$self->{strS3Command} rm --recursive s3://pgbackrest-dev"); +} + +#################################################################################################################################### +# run +#################################################################################################################################### +sub run +{ + my $self = shift; + + # Initialize the driver + my $oS3 = $self->initS3(); + my $oStorage = new pgBackRest::Storage::Local('', $oS3); + + # Test variables + my $strFile = 'file.txt'; + my $strFileContent = 'TESTDATA'; + my $iFileLength = length($strFileContent); + + ################################################################################################################################ + if ($self->begin('exists()')) + { + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult(sub {$oStorage->exists($strFile)}, false, 'root file does not exist'); + + #--------------------------------------------------------------------------------------------------------------------------- + storageTest()->put($strFile, $strFileContent); + executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev"); + + $self->testResult(sub {$oStorage->exists($strFile)}, true, 'root file exists'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult(sub {$oStorage->pathExists('/path/to')}, false, 'sub path does not exist'); + $self->testResult(sub {$oStorage->exists("/path/to/${strFile}")}, false, 'sub file does not exist'); + + #--------------------------------------------------------------------------------------------------------------------------- + executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev/path/to/${strFile}"); + + $self->testResult(sub {$oStorage->pathExists('/path/to')}, true, 'sub path exists'); + # $oStorage->pathExists('/path/to'); + $self->testResult(sub {$oStorage->exists("/path/to/${strFile}")}, true, 'sub file exists'); + } + + ################################################################################################################################ + if ($self->begin('manifest()')) + { + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult(sub {$oStorage->manifest('')}, '{. => {type => d}}', 'no files'); + + #--------------------------------------------------------------------------------------------------------------------------- + storageTest()->put($strFile, $strFileContent); + storageTest()->put("${strFile}2", $strFileContent . '2'); + + executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev"); + executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2"); + + $self->testResult( + sub {$oStorage->manifest('')}, + '{. => {type => d}, file.txt => {size => 8, type => f}, path => {type => d}, path/to => {type => d},' . + ' path/to/file.txt2 => {size => 9, type => f}}', + 'root path'); + $self->testResult( + sub {$oStorage->manifest('/path/to')}, '{. => {type => d}, file.txt2 => {size => 9, type => f}}', 'sub path'); + } + + ################################################################################################################################ + if ($self->begin('list()')) + { + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult( + sub {$oStorage->list('')}, '[undef]', 'no files'); + + #--------------------------------------------------------------------------------------------------------------------------- + storageTest()->put($strFile, $strFileContent); + storageTest()->put("${strFile}2", $strFileContent . '2'); + + executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev"); + executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2"); + + $self->testResult(sub {$oStorage->list('')}, '(file.txt, path)', 'root path'); + $self->testResult(sub {$oStorage->list('/path/to')}, 'file.txt2', 'sub path'); + } + + ################################################################################################################################ + if ($self->begin('remove()')) + { + #--------------------------------------------------------------------------------------------------------------------------- + $oStorage->put($strFile, $strFileContent); + $oStorage->put("/path/to/${strFile}2", $strFileContent); + $oStorage->put("/path/to/${strFile}3", $strFileContent); + $oStorage->put("/path/to/${strFile}4", $strFileContent); + + $self->testResult( + sub {$oStorage->manifest('/')}, + '{. => {type => d}, file.txt => {size => 8, type => f}, path => {type => d}, path/to => {type => d},' . + ' path/to/file.txt2 => {size => 8, type => f}, path/to/file.txt3 => {size => 8, type => f},' . + ' path/to/file.txt4 => {size => 8, type => f}}', + 'check manifest'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult(sub {$oStorage->remove('/path/to', {bRecurse => true})}, true, 'remove subpath'); + + $self->testResult( + sub {$oStorage->manifest('/')}, + '{. => {type => d}, file.txt => {size => 8, type => f}}', 'check manifest'); + + #--------------------------------------------------------------------------------------------------------------------------- + $self->testResult(sub {$oStorage->remove($strFile)}, true, 'remove file'); + + $self->testResult(sub {$oStorage->manifest('/')}, '{. => {type => d}}', 'check manifest'); + } + + ################################################################################################################################ + if ($self->begin('info()')) + { + #--------------------------------------------------------------------------------------------------------------------------- + storageTest()->put($strFile, $strFileContent); + storageTest()->put("${strFile}2", $strFileContent . '2'); + + executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev"); + executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev"); + executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2"); + + $self->testResult(sub {$oStorage->info($strFile)->size()}, 8, 'file size'); + $self->testResult(sub {$oStorage->info("/path/to/${strFile}2")->size()}, 9, 'file 2 size'); + } + + ################################################################################################################################ + if ($self->begin('openRead() && S3::FileRead')) + { + # Create a random 1mb file + my $strRandomFile = $self->testPath() . '/random1mb.bin'; + executeTest("dd if=/dev/urandom of=${strRandomFile} bs=1024k count=1", {bSuppressStdErr => true}); + my $strRandom = ${storageTest()->get($strRandomFile)}; + + executeTest("$self->{strS3Command} cp ${strRandomFile} s3://pgbackrest-dev/path/to/${strFile}"); + + #--------------------------------------------------------------------------------------------------------------------------- + my $tBuffer; + my $oFileRead = $self->testResult(sub {$oS3->openRead("/path/to/${strFile}")}, '[object]', 'open read'); + $self->testResult(sub {$oFileRead->read(\$tBuffer, 524288)}, 524288, ' read half'); + $self->testResult(sub {$oFileRead->read(\$tBuffer, 524288)}, 524288, ' read half'); + $self->testResult(sub {$oFileRead->read(\$tBuffer, 512)}, 0, ' read 0'); + $self->testResult(length($tBuffer), 1048576, ' check length'); + $self->testResult(sha1_hex($tBuffer), sha1_hex($strRandom), ' check hash'); + } + + ################################################################################################################################ + if ($self->begin('openWrite() S3::FileWrite')) + { + # Create a random 1mb file + my $strRandomFile = $self->testPath() . '/random1mb.bin'; + executeTest("dd if=/dev/urandom of=${strRandomFile} bs=1024k count=1", {bSuppressStdErr => true}); + my $strRandom = ${storageTest()->get($strRandomFile)}; + + #--------------------------------------------------------------------------------------------------------------------------- + my $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write'); + $self->testResult(sub {$oFileWrite->close()}, true, ' close without writing'); + + #--------------------------------------------------------------------------------------------------------------------------- + $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write'); + $self->testResult(sub {$oFileWrite->write()}, 0, ' write undef'); + $self->testResult(sub {$oFileWrite->write(\$strFileContent)}, $iFileLength, ' write'); + $self->testResult(sub {$oFileWrite->close()}, true, ' close'); + + #--------------------------------------------------------------------------------------------------------------------------- + $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write'); + + for (my $iIndex = 1; $iIndex <= 17; $iIndex++) + { + $self->testResult(sub {$oFileWrite->write(\$strRandom)}, 1024 * 1024, ' write 1mb'); + } + + $self->testResult(sub {$oFileWrite->close()}, true, ' close'); + } +} + +1;