S3 repository support.

.travis.yml

@@ -37,6 +37,14 @@ env:
 
 before_install:
   - sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl python-pip
+  - |
+    # Install & Configure AWS CLI
+    pip install --upgrade --user awscli
+    aws configure set region us-east-1
+    aws configure set aws_access_key_id accessKey1
+    aws configure set aws_secret_access_key verySecretKey1
+    aws help --version
+    aws configure list
   - |
     # Build Devel::Cover
     git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl

README.md

@@ -72,6 +72,10 @@ Tablespaces are fully supported and on restore tablespaces can be remapped to an
 
 File and directory links are supported for any file or directory in the PostgreSQL cluster. When restoring it is possible to restore all links to their original locations, remap some or all links, or restore some or all links as normal files or directories within the cluster directory.
 
+### Amazon S3 Support
+
+pgBackRest repositories can be stored on Amazon S3 to allow for virtually unlimited capacity and retention.
+
 ### Compatibility with PostgreSQL >= 8.3
 
 pgBackRest includes support for versions down to 8.3, since older versions of PostgreSQL are still regularly utilized.

bin/pgbackrest

@@ -63,7 +63,7 @@ local $EVAL_ERROR = undef; eval
         optionSet(OPTION_LOG_LEVEL_STDERR, PROTOCOL, true);
         logLevelSet(OFF, OFF, optionGet(OPTION_LOG_LEVEL_STDERR));
 
-        if (optionTest(OPTION_TYPE, BACKUP) && !-e optionGet(OPTION_REPO_PATH))
+        if (optionTest(OPTION_TYPE, BACKUP) && !optionTest(OPTION_REPO_TYPE, REPO_TYPE_S3) && !-e optionGet(OPTION_REPO_PATH))
         {
             confess &log(ERROR, 'repo-path \'' . optionGet(OPTION_REPO_PATH) . '\' does not exist', ERROR_PATH_MISSING);
         }
@@ -169,7 +169,7 @@ local $EVAL_ERROR = undef; eval
     require pgBackRest::Protocol::Storage::Helper;
     pgBackRest::Protocol::Storage::Helper->import();
 
-    if (isRepoLocal() && !storageRepo()->pathExists(''))
+    if (isRepoLocal() && !optionTest(OPTION_REPO_TYPE, REPO_TYPE_S3) && !storageRepo()->pathExists(''))
     {
         confess &log(ERROR, 'repo-path \'' . optionGet(OPTION_REPO_PATH) . '\' does not exist', ERROR_PATH_MISSING);
     }

doc/lib/BackRestDoc/Common/DocExecute.pm

@@ -572,6 +572,7 @@ sub backrestConfig
             my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}});
             delete($$oConfigClean{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_LEVEL_STDERR});
             delete($$oConfigClean{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_TIMESTAMP});
+            delete($$oConfigClean{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_VERIFY_SSL});
 
             if (keys(%{$$oConfigClean{&CONFIG_SECTION_GLOBAL}}) == 0)
             {

doc/xml/index.xml

@@ -124,6 +124,12 @@
             <p>File and directory links are supported for any file or directory in the <postgres/> cluster.  When restoring it is possible to restore all links to their original locations, remap some or all links, or restore some or all links as normal files or directories within the cluster directory.</p>
         </section>
 
+        <section id="s3-support">
+            <title>Amazon S3 Support</title>
+
+            <p><backrest/> repositories can be stored on <proper>Amazon S3</proper> to allow for virtually unlimited capacity and retention.</p>
+        </section>
+
         <section id="postgres-compatibility">
             <title>Compatibility with <postgres/> >= 8.3</title>
 

doc/xml/reference.xml

@@ -229,6 +229,71 @@
                         <example>/backup/db/backrest</example>
                     </config-key>
 
+                    <!-- CONFIG - GENERAL SECTION - REPO-S3-KEY KEY -->
+                    <config-key id="repo-s3-key" name="S3 Repository Access Key">
+                        <summary>S3 repository access key.</summary>
+
+                        <text>AWS key used to access this bucket.</text>
+
+                        <example>AKIAIOSFODNN7EXAMPLE</example>
+                    </config-key>
+
+                    <!-- CONFIG - GENERAL SECTION - REPO-S3-KEY-SECRET KEY -->
+                    <config-key id="repo-s3-key-secret" name="S3 Repository Secret Access Key">
+                        <summary>S3 repository secret access key.</summary>
+
+                        <text>AWS secret key used to access this bucket.</text>
+
+                        <example>wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY</example>
+                    </config-key>
+
+                    <!-- CONFIG - GENERAL SECTION - REPO-S3-BUCKET KEY -->
+                    <config-key id="repo-s3-bucket" name="S3 Repository Bucket">
+                        <summary>S3 repository bucket.</summary>
+
+                        <text>S3 bucket used to store the repository.
+
+                        <backrest/> repositories can be stored in the bucket root by setting <br-option>repo-path=/</br-option> but it is usually best to specify a prefix, such as <path>/repo</path>, so logs and other AWS generated content can also be stored in the bucket.</text>
+
+                        <example>db-backup</example>
+                    </config-key>
+
+                    <!-- CONFIG - GENERAL SECTION - REPO-S3-ENDPOINT KEY -->
+                    <config-key id="repo-s3-endpoint" name="S3 Repository Endpoint">
+                        <summary>S3 repository endpoint.</summary>
+
+                        <text>The AWS end point should be valid for the selected region.</text>
+
+                        <example>s3.amazonaws.com</example>
+                    </config-key>
+
+                    <!-- CONFIG - GENERAL SECTION - REPO-S3-HOST KEY -->
+                    <config-key id="repo-s3-host" name="S3 Repository Host">
+                        <summary>S3 repository host.</summary>
+
+                        <text>Connect to a host other than the end point.  This is typically used for testing.</text>
+
+                        <example>127.0.0.1</example>
+                    </config-key>
+
+                    <!-- CONFIG - GENERAL SECTION - REPO-S3-REGION KEY -->
+                    <config-key id="repo-s3-region" name="S3 Repository Region">
+                        <summary>S3 repository region.</summary>
+
+                        <text>The AWS region where the bucket was created.</text>
+
+                        <example>us-east-1</example>
+                    </config-key>
+
+                    <!-- CONFIG - GENERAL SECTION - REPO-S3-VERIFY-SSL KEY -->
+                    <config-key id="repo-s3-verify-ssl" name="S3 Repository Verify SSL">
+                        <summary>Verify S3 server certificate.</summary>
+
+                        <text>Disables verification of the S3 server certificate.  This should only be used for testing or other scenarios where a certificate has been self-signed.</text>
+
+                        <example>n</example>
+                    </config-key>
+
                     <!-- CONFIG - GENERAL SECTION - REPO-TYPE KEY -->
                     <config-key id="repo-type" name="Repository Type">
                         <summary>Type of storage used for the repository.</summary>
@@ -237,6 +302,7 @@
                         <ul>
                             <li><id>cifs</id> - Like <id>posix</id>, but disables links and directory fsyncs</li>
                             <li><id>posix</id> - Posix-compliant file systems</li>
+                            <li><id>s3</id> - AWS Simple Storage Service</li>
                         </ul></text>
 
                         <example>cifs</example>

doc/xml/release.xml

@@ -186,6 +186,16 @@
                     </release-item>
                 </release-bug-list>
 
+                <release-feature-list>
+                    <release-item>
+                        <release-item-contributor-list>
+                            <release-item-reviewer id="shang.cynthia"/>
+                        </release-item-contributor-list>
+
+                        <p><proper>Amazon S3</proper> repository support.</p>
+                    </release-item>
+                </release-feature-list>
+
                 <release-refactor-list>
                     <release-item>
                         <release-item-contributor-list>

doc/xml/user-guide.xml

@@ -170,6 +170,10 @@
     <section id="installation">
         <title>Installation</title>
 
+        <!-- Create S3 server first to allow it time to boot before being used -->
+        <host-add name="demo-bucket.s3.amazonaws.com" user="root" image="{[image-repo]}:{[host-os]}-s3-server" os="{[host-os]}">
+        </host-add>
+
         <host-add name="{[host-db-master]}" user="{[host-db-master-user]}" image="{[host-db-master-image]}" os="{[host-os]}" mount="{[host-db-master-mount]}">
             <execute user="root">
                 <exe-cmd>mkdir /root/pgbackrest-release-{[version]}</exe-cmd>
@@ -1321,6 +1325,55 @@
         </execute-list>
     </section>
 
+    <!-- SECTION => S3-SUPPORT -->
+    <section id="s3-support" depend="/quickstart/configure-archiving">
+        <title>S3 Support</title>
+
+        <p><backrest/> supports storing repositories in <proper>Amazon S3</proper>.</p>
+
+        <backrest-config host="{[host-db-master]}" file="{[backrest-config-demo]}">
+            <title>Configure <proper>S3</proper></title>
+
+            <backrest-config-option section="global" key="repo-type">s3</backrest-config-option>
+            <backrest-config-option section="global" key="repo-path">/</backrest-config-option>
+            <backrest-config-option section="global" key="repo-s3-key">accessKey1</backrest-config-option>
+            <backrest-config-option section="global" key="repo-s3-key-secret">verySecretKey1</backrest-config-option>
+            <backrest-config-option section="global" key="repo-s3-bucket">demo-bucket</backrest-config-option>
+            <backrest-config-option section="global" key="repo-s3-endpoint">s3.amazonaws.com</backrest-config-option>
+            <backrest-config-option section="global" key="repo-s3-region">us-east-1</backrest-config-option>
+            <backrest-config-option section="global" key="repo-s3-verify-ssl">n</backrest-config-option>
+
+            <backrest-config-option section="global" key="process-max">4</backrest-config-option>
+        </backrest-config>
+
+        <p>Commands are run exactly as if the repository were stored on a local disk.</p>
+
+        <execute-list host="{[host-db-master]}">
+            <title>Create the stanza</title>
+
+            <execute user="{[host-user]}">
+                <exe-cmd>aws s3 --no-verify-ssl mb s3://demo-bucket 2>&amp;1</exe-cmd>
+            </execute>
+
+            <execute output="y">
+                <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-create</exe-cmd>
+                <exe-highlight>completed successfully</exe-highlight>
+            </execute>
+        </execute-list>
+
+        <p>File creation time in <proper>S3</proper> is relatively slow so commands benefit by increasing <br-option>process-max</br-option> to parallelize file creation.</p>
+
+        <execute-list host="{[host-db-master]}">
+            <title>Backup the {[postgres-cluster-demo]} cluster</title>
+
+            <execute output="y">
+                <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]}
+                    --log-level-console=info backup</exe-cmd>
+                <exe-highlight>no prior backup exists|full backup size</exe-highlight>
+            </execute>
+        </execute-list>
+    </section>
+
     <!-- SECTION => BACKUP HOST -->
     <section id="backup-host" depend="/quickstart/configure-archiving">
         <title>Dedicated Backup Host</title>

lib/pgBackRest/Common/Http/Client.pm

@@ -0,0 +1,283 @@
+####################################################################################################################################
+# HTTP Client
+####################################################################################################################################
+package pgBackRest::Common::Http::Client;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+    our @EXPORT = qw();
+use IO::Socket::SSL;
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Io::Buffered;
+use pgBackRest::Common::Log;
+use pgBackRest::Common::String;
+use pgBackRest::Common::Xml;
+use pgBackRest::Common::Http::Common;
+
+####################################################################################################################################
+# Constants
+####################################################################################################################################
+use constant HTTP_VERB_GET                                          => 'GET';
+    push @EXPORT, qw(HTTP_VERB_GET);
+use constant HTTP_VERB_POST                                         => 'POST';
+    push @EXPORT, qw(HTTP_VERB_POST);
+use constant HTTP_VERB_PUT                                          => 'PUT';
+    push @EXPORT, qw(HTTP_VERB_PUT);
+
+use constant HTTP_HEADER_CONTENT_LENGTH                               => 'content-length';
+    push @EXPORT, qw(HTTP_HEADER_CONTENT_LENGTH);
+use constant HTTP_HEADER_TRANSFER_ENCODING                            => 'transfer-encoding';
+    push @EXPORT, qw(HTTP_HEADER_TRANSFER_ENCODING);
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+our @ISA = ();                                                      ## no critic (ClassHierarchies::ProhibitExplicitISA)
+
+sub new
+{
+    my $class = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strHost,
+        $strVerb,
+        $strUri,
+        $hQuery,
+        $hRequestHeader,
+        $rstrRequestBody,
+        $iProtocolTimeout,
+        $lBufferMax,
+        $bVerifySsl,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->new', \@_,
+            {name => 'strHost', trace => true},
+            {name => 'strVerb', trace => true},
+            {name => 'strUri', optional => true, default => qw(/), trace => true},
+            {name => 'hQuery', optional => true, trace => true},
+            {name => 'hRequestHeader', optional => true, trace => true},
+            {name => 'rstrRequestBody', optional => true, trace => true},
+            {name => 'iProtocolTimeout', optional => true, default => 30, trace => true},
+            {name => 'lBufferMax', optional => true, default => 32768, trace => true},
+            {name => 'bVerifySsl', optional => true, default => true, trace => true},
+        );
+
+    # Connect to the server
+    my $oSocket = IO::Socket::SSL->new(
+        PeerHost => $strHost, PeerPort => 'https', SSL_verify_mode => $bVerifySsl ? SSL_VERIFY_PEER : SSL_VERIFY_NONE);
+
+    if (!defined($oSocket))
+    {
+        logErrorResult(ERROR_PROTOCOL, "unable to connect $!", $SSL_ERROR);
+    }
+
+    # Create the buffered IO object
+    my $self = new pgBackRest::Common::Io::Buffered(
+        new pgBackRest::Common::Io::Handle('httpClient', $oSocket, $oSocket), $iProtocolTimeout, $lBufferMax);
+
+    # Bless with the class
+    @ISA = $self->isA();                                            ## no critic (ClassHierarchies::ProhibitExplicitISA)
+    bless $self, $class;
+
+    # Store socket
+    $self->{oSocket} = $oSocket;
+
+    # Generate the query string
+    my $strQuery = httpQuery($hQuery);
+
+    # Construct the request headers
+    $self->{strRequestHeader} = "${strVerb} ${strUri}?${strQuery} HTTP/1.1" . "\r\n";
+
+    foreach my $strHeader (sort(keys(%{$hRequestHeader})))
+    {
+        $self->{strRequestHeader} .= "${strHeader}: $hRequestHeader->{$strHeader}\r\n";
+    }
+
+    $self->{strRequestHeader} .= "\r\n";
+
+    # Write request headers
+    $self->write(\$self->{strRequestHeader});
+
+    # Write content
+    if (defined($rstrRequestBody))
+    {
+        my $iTotalSize = length($$rstrRequestBody);
+        my $iTotalSent = 0;
+
+        # Write the request body in buffer-sized chunks
+        do
+        {
+            my $strBufferWrite = substr($$rstrRequestBody, $iTotalSent, $lBufferMax);
+            $iTotalSent += $self->write(\$strBufferWrite);
+        } while ($iTotalSent < $iTotalSize);
+    }
+
+    # Read response code
+    ($self->{strResponseProtocol}, $self->{iResponseCode}, $self->{strResponseMessage}) = split(' ', trim($self->readLine()));
+
+    # Read the response headers
+    $self->{iContentLength} = undef;
+
+    my $strResponseHeader = '';
+    my $strHeader = trim($self->readLine());
+
+    while ($strHeader ne '')
+    {
+        # Validate header
+        $strResponseHeader .= "${strHeader}\n";
+
+        my $iColonPos = index($strHeader, ':');
+
+        if ($iColonPos == -1)
+        {
+            confess &log(ERROR, "http header '${strHeader}' requires colon separator", ERROR_PROTOCOL);
+        }
+
+        # Parse header
+        my $strHeaderKey = lc(substr($strHeader, 0, $iColonPos));
+        my $strHeaderValue = trim(substr($strHeader, $iColonPos + 1));
+
+        # Store the header
+        $self->{hResponseHeader}{$strHeaderKey} = $strHeaderValue;
+
+        # Process content length
+        if ($strHeaderKey eq HTTP_HEADER_CONTENT_LENGTH)
+        {
+            $self->{iContentLength} = $strHeaderValue + 0;
+            $self->{iContentRemaining} = $self->{iContentLength};
+        }
+        # Process transfer encoding (only chunked is supported)
+        elsif ($strHeaderKey eq HTTP_HEADER_TRANSFER_ENCODING)
+        {
+            if ($strHeaderValue eq 'chunked')
+            {
+                $self->{iContentLength} = -1;
+            }
+            else
+            {
+                confess &log(ERROR, "invalid value '${strHeaderValue} for http header '${strHeaderKey}'", ERROR_PROTOCOL);
+            }
+        }
+
+        # Read next header
+        $strHeader = trim($self->readLine());
+    }
+
+    # Test response code
+    if ($self->{iResponseCode} == 200)
+    {
+        # Content length should have been defined either by content-length or transfer encoding
+        if (!defined($self->{iContentLength}))
+        {
+            confess &log(ERROR,
+                HTTP_HEADER_CONTENT_LENGTH . ' or ' . HTTP_HEADER_TRANSFER_ENCODING . ' must be defined', ERROR_PROTOCOL);
+        }
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'self', value => $self}
+    );
+}
+
+####################################################################################################################################
+# read - read content from http stream
+####################################################################################################################################
+sub read
+{
+    my $self = shift;
+    my $rtBuffer = shift;
+    my $iRequestSize = shift;
+
+    # Make sure request size is not larger than what remains to be read
+    $iRequestSize = $iRequestSize < $self->{iContentRemaining} ? $iRequestSize : $self->{iContentRemaining};
+    $self->{iContentRemaining} -= $iRequestSize;
+
+    return $self->SUPER::read($rtBuffer, $iRequestSize, true);
+}
+
+####################################################################################################################################
+# close/DESTROY - close the HTTP connection
+####################################################################################################################################
+sub close
+{
+    my $self = shift;
+
+    # Only close if the socket is open
+    if (defined($self->{oSocket}))
+    {
+        $self->{oSocket}->close();
+        undef($self->{oSocket});
+    }
+}
+
+sub DESTROY {shift->close()}
+
+####################################################################################################################################
+# responseBody - return the entire body of the response in a buffer
+####################################################################################################################################
+sub responseBody
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->responseBody'
+        );
+
+    my $strResponseBody = undef;
+
+    # Nothing to do if content length is 0
+    if ($self->{iContentLength} != 0)
+    {
+        while (1)
+        {
+            # Read chunk length
+            my $strChunkLength = trim($self->readLine());
+            my $iChunkLength = hex($strChunkLength);
+
+            # Exit if chunk length is 0
+            last if ($iChunkLength == 0);
+
+            # Read the chunk and consume the terminating LF
+            $self->SUPER::read(\$strResponseBody, $iChunkLength, true);
+            $self->readLine();
+        };
+
+        $self->close();
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'rstrResponseBody', value => \$strResponseBody, trace => true}
+    );
+}
+
+####################################################################################################################################
+# Properties.
+####################################################################################################################################
+sub contentLength {shift->{iContentLength}}                         # Content length if available (-1 means not known yet)
+sub responseCode {shift->{iResponseCode}}
+sub responseHeader {shift->{hResponseHeader}}
+sub responseMessage {shift->{strResponseMessage}}
+sub responseProtocol {shift->{strResponseProtocol}}
+
+1;

lib/pgBackRest/Common/Http/Common.pm

@@ -0,0 +1,106 @@
+####################################################################################################################################
+# HTTP Common
+####################################################################################################################################
+package pgBackRest::Common::Http::Common;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+    our @EXPORT = qw();
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# httpQuery - encode an HTTP query from a hash
+####################################################################################################################################
+sub httpQuery
+{
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $hQuery,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '::httpQuery', \@_,
+            {name => 'hQuery', required => false, trace => true},
+        );
+
+    # Generate the query string
+    my $strQuery = '';
+
+    # If a hash (the normal case)
+    if (ref($hQuery))
+    {
+        foreach my $strParam (sort(keys(%{$hQuery})))
+        {
+            # Parameters may not be defined - this is OK
+            if (defined($hQuery->{$strParam}))
+            {
+                $strQuery .= ($strQuery eq '' ? '' : '&') . $strParam . '=' . httpUriEncode($hQuery->{$strParam});
+            }
+        }
+    }
+    # Else query string was passed directly as a scalar
+    elsif (defined($hQuery))
+    {
+        $strQuery = $hQuery;
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'strQuery', value => $strQuery, trace => true}
+    );
+}
+
+push @EXPORT, qw(httpQuery);
+
+####################################################################################################################################
+# httpUriEncode - encode query values to conform with URI specs
+####################################################################################################################################
+sub httpUriEncode
+{
+    my $strString = shift;
+
+    # Only encode if source string is defined
+    my $strEncodedString;
+
+    if (defined($strString))
+    {
+        # Iterate all characters in the string
+        for (my $iIndex = 0; $iIndex < length($strString); $iIndex++)
+        {
+            my $cChar = substr($strString, $iIndex, 1);
+
+            # These characters are reproduced verbatim
+            if (($cChar ge 'A' && $cChar le 'Z') || ($cChar ge 'a' && $cChar le 'z') || ($cChar ge '0' && $cChar le '9') ||
+                $cChar eq '_' || $cChar eq '-' || $cChar eq '~' || $cChar eq '.')
+            {
+                $strEncodedString .= $cChar;
+            }
+            # Forward slash is encoded
+            elsif ($cChar eq '/')
+            {
+                $strEncodedString .= '%2F';
+            }
+            # All other characters are hex-encoded
+            else
+            {
+                $strEncodedString .= sprintf('%%%02X', ord($cChar));
+            }
+        }
+    }
+
+    return $strEncodedString;
+}
+
+push @EXPORT, qw(httpUriEncode);
+
+1;

lib/pgBackRest/Common/Xml.pm

@@ -0,0 +1,151 @@
+####################################################################################################################################
+# XML Helper Functions
+####################################################################################################################################
+package pgBackRest::Common::Xml;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+    our @EXPORT = qw();
+use XML::LibXML;
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# xmlParse - parse a string into an xml document and return the root node
+####################################################################################################################################
+use constant XML_HEADER                                             => '<?xml version="1.0" encoding="UTF-8"?>';
+    push @EXPORT, qw(XML_HEADER);
+
+####################################################################################################################################
+# xmlParse - parse a string into an xml document and return the root node
+####################################################################################################################################
+sub xmlParse
+{
+    my $rstrXml = shift;
+
+    my $oXml = XML::LibXML->load_xml(string => $rstrXml)->documentElement();
+
+    return $oXml;
+}
+
+push @EXPORT, qw(xmlParse);
+
+####################################################################################################################################
+# xmlTagChildren - get all children that match the tag
+####################################################################################################################################
+sub xmlTagChildren
+{
+    my $oXml = shift;
+    my $strTag = shift;
+
+    return $oXml->getChildrenByTagName($strTag);
+}
+
+push @EXPORT, qw(xmlTagChildren);
+
+####################################################################################################################################
+# xmlTagText - get the text content for a tag, error if the tag is required and does not exist
+####################################################################################################################################
+sub xmlTagText
+{
+    my $oXml = shift;
+    my $strTag = shift;
+    my $bRequired = shift;
+    # my $strDefault = shift;
+
+    # Get the tag or tags
+    my @oyTag = $oXml->getElementsByTagName($strTag);
+
+    # Error if the tag does not exist and is required
+    if (@oyTag > 1)
+    {
+        confess &log(ERROR, @oyTag . " '${strTag}' tag(s) exist, but only one was expected", ERROR_FORMAT);
+    }
+    elsif (@oyTag == 0)
+    {
+        if (!defined($bRequired) || $bRequired)
+        {
+            confess &log(ERROR, "tag '${strTag}' does not exist", ERROR_FORMAT);
+        }
+    }
+    else
+    {
+        return $oyTag[0]->textContent();
+    }
+
+    return;
+}
+
+push @EXPORT, qw(xmlTagText);
+
+####################################################################################################################################
+# xmlTagBool - get the boolean content for a tag, error if the tag is required and does not exist or is not boolean
+####################################################################################################################################
+sub xmlTagBool
+{
+    my $oXml = shift;
+    my $strTag = shift;
+    my $bRequired = shift;
+    # my $strDefault = shift;
+
+    # Test content for boolean value
+    my $strContent = xmlTagText($oXml, $strTag, $bRequired);
+
+    if (defined($strContent))
+    {
+        if ($strContent eq 'true')
+        {
+            return true;
+        }
+        elsif ($strContent eq 'false')
+        {
+            return false;
+        }
+        else
+        {
+            confess &log(ERROR, "invalid boolean value '${strContent}' for tag '${strTag}'", ERROR_FORMAT);
+        }
+    }
+
+    return;
+}
+
+push @EXPORT, qw(xmlTagBool);
+
+####################################################################################################################################
+# xmlTagInt - get the integer content for a tag, error if the tag is required and does not exist or is not an integer
+####################################################################################################################################
+sub xmlTagInt
+{
+    my $oXml = shift;
+    my $strTag = shift;
+    my $bRequired = shift;
+    # my $strDefault = shift;
+
+    # Test content for boolean value
+    my $iContent = xmlTagText($oXml, $strTag, $bRequired);
+
+    if (defined($iContent))
+    {
+        eval
+        {
+            $iContent = $iContent + 0;
+            return 1;
+        }
+        or do
+        {
+            confess &log(ERROR, "invalid integer value '${iContent}' for tag '${strTag}'", ERROR_FORMAT);
+        }
+    }
+
+    return $iContent;
+}
+
+push @EXPORT, qw(xmlTagInt);
+
+1;

lib/pgBackRest/Config/Config.pm

@@ -97,6 +97,8 @@ use constant REPO_TYPE_CIFS                                         => 'cifs';
     push @EXPORT, qw(REPO_TYPE_CIFS);
 use constant REPO_TYPE_POSIX                                        => 'posix';
     push @EXPORT, qw(REPO_TYPE_POSIX);
+use constant REPO_TYPE_S3                                           => 's3';
+    push @EXPORT, qw(REPO_TYPE_S3);
 
 ####################################################################################################################################
 # INFO Output Constants
@@ -298,6 +300,22 @@ use constant OPTION_REPO_PATH                                       => 'repo-pat
 use constant OPTION_REPO_TYPE                                       => 'repo-type';
     push @EXPORT, qw(OPTION_REPO_TYPE);
 
+# Repository S3
+use constant OPTION_REPO_S3_KEY                                     => 'repo-s3-key';
+    push @EXPORT, qw(OPTION_REPO_S3_KEY);
+use constant OPTION_REPO_S3_KEY_SECRET                              => 'repo-s3-key-secret';
+    push @EXPORT, qw(OPTION_REPO_S3_KEY_SECRET);
+use constant OPTION_REPO_S3_BUCKET                                  => 'repo-s3-bucket';
+    push @EXPORT, qw(OPTION_REPO_S3_BUCKET);
+use constant OPTION_REPO_S3_ENDPOINT                                => 'repo-s3-endpoint';
+    push @EXPORT, qw(OPTION_REPO_S3_ENDPOINT);
+use constant OPTION_REPO_S3_HOST                                    => 'repo-s3-host';
+    push @EXPORT, qw(OPTION_REPO_S3_HOST);
+use constant OPTION_REPO_S3_REGION                                  => 'repo-s3-region';
+    push @EXPORT, qw(OPTION_REPO_S3_REGION);
+use constant OPTION_REPO_S3_VERIFY_SSL                              => 'repo-s3-verify-ssl';
+    push @EXPORT, qw(OPTION_REPO_S3_VERIFY_SSL);
+
 # Log level
 use constant OPTION_LOG_LEVEL_CONSOLE                               => 'log-level-console';
     push @EXPORT, qw(OPTION_LOG_LEVEL_CONSOLE);
@@ -486,6 +504,8 @@ use constant OPTION_DEFAULT_REPO_LINK                               => true;
     push @EXPORT, qw(OPTION_DEFAULT_REPO_LINK);
 use constant OPTION_DEFAULT_REPO_PATH                               => '/var/lib/' . BACKREST_EXE;
     push @EXPORT, qw(OPTION_DEFAULT_REPO_PATH);
+use constant OPTION_DEFAULT_REPO_S3_VERIFY_SSL                      => true;
+    push @EXPORT, qw(OPTION_DEFAULT_REPO_S3_VERIFY_SSL);
 use constant OPTION_DEFAULT_REPO_TYPE                               => REPO_TYPE_POSIX;
     push @EXPORT, qw(OPTION_DEFAULT_REPO_TYPE);
 use constant OPTION_DEFAULT_SPOOL_PATH                              => '/var/spool/' . BACKREST_EXE;
@@ -1271,6 +1291,53 @@ my %oOptionRule =
         },
     },
 
+    &OPTION_REPO_S3_BUCKET =>
+    {
+        &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
+        &OPTION_RULE_DEPEND =>
+        {
+            &OPTION_RULE_DEPEND_OPTION  => OPTION_REPO_TYPE,
+            &OPTION_RULE_DEPEND_VALUE   => REPO_TYPE_S3,
+        },
+        &OPTION_RULE_COMMAND => OPTION_REPO_TYPE,
+    },
+
+    &OPTION_REPO_S3_KEY =>
+    {
+        &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
+        &OPTION_RULE_SECURE => true,
+        &OPTION_RULE_REQUIRED => false,
+        &OPTION_RULE_DEPEND =>
+        {
+            &OPTION_RULE_DEPEND_OPTION  => OPTION_REPO_TYPE,
+            &OPTION_RULE_DEPEND_VALUE   => REPO_TYPE_S3,
+        },
+        &OPTION_RULE_COMMAND => OPTION_REPO_TYPE,
+    },
+
+    &OPTION_REPO_S3_KEY_SECRET => OPTION_REPO_S3_KEY,
+
+    &OPTION_REPO_S3_ENDPOINT => OPTION_REPO_S3_BUCKET,
+
+    &OPTION_REPO_S3_HOST =>
+    {
+        &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
+        &OPTION_RULE_REQUIRED  => false,
+        &OPTION_RULE_DEPEND => OPTION_REPO_S3_BUCKET,
+        &OPTION_RULE_COMMAND => OPTION_REPO_TYPE,
+    },
+
+    &OPTION_REPO_S3_REGION => OPTION_REPO_S3_BUCKET,
+
+    &OPTION_REPO_S3_VERIFY_SSL =>
+    {
+        &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
+        &OPTION_RULE_TYPE => OPTION_TYPE_BOOLEAN,
+        &OPTION_RULE_DEFAULT => OPTION_DEFAULT_REPO_S3_VERIFY_SSL,
+        &OPTION_RULE_COMMAND => OPTION_REPO_TYPE,
+        &OPTION_RULE_DEPEND => OPTION_REPO_S3_BUCKET,
+    },
+
     &OPTION_REPO_TYPE =>
     {
         &OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
@@ -1280,6 +1347,7 @@ my %oOptionRule =
         {
             &REPO_TYPE_CIFS     => true,
             &REPO_TYPE_POSIX    => true,
+            &REPO_TYPE_S3       => true,
         },
         &OPTION_RULE_COMMAND =>
         {

lib/pgBackRest/Config/ConfigHelpData.pm

@@ -608,6 +608,87 @@ my $oConfigHelpData =
                     "need, though of course requirements will likely change over time as your database evolves."
         },
 
+        # REPO-S3-BUCKET Option Help
+        #---------------------------------------------------------------------------------------------------------------------------
+        'repo-s3-bucket' =>
+        {
+            section => 'general',
+            summary =>
+                "S3 repository bucket.",
+            description =>
+                "S3 bucket used to store the repository.\n" .
+                "\n" .
+                "pgBackRest repositories can be stored in the bucket root by setting repo-path=/ but it is usually best to " .
+                    "specify a prefix, such as /repo, so logs and other AWS generated content can also be stored in the bucket."
+        },
+
+        # REPO-S3-ENDPOINT Option Help
+        #---------------------------------------------------------------------------------------------------------------------------
+        'repo-s3-endpoint' =>
+        {
+            section => 'general',
+            summary =>
+                "S3 repository endpoint.",
+            description =>
+                "The AWS end point should be valid for the selected region."
+        },
+
+        # REPO-S3-HOST Option Help
+        #---------------------------------------------------------------------------------------------------------------------------
+        'repo-s3-host' =>
+        {
+            section => 'general',
+            summary =>
+                "S3 repository host.",
+            description =>
+                "Connect to a host other than the end point. This is typically used for testing."
+        },
+
+        # REPO-S3-KEY Option Help
+        #---------------------------------------------------------------------------------------------------------------------------
+        'repo-s3-key' =>
+        {
+            section => 'general',
+            summary =>
+                "S3 repository access key.",
+            description =>
+                "AWS key used to access this bucket."
+        },
+
+        # REPO-S3-KEY-SECRET Option Help
+        #---------------------------------------------------------------------------------------------------------------------------
+        'repo-s3-key-secret' =>
+        {
+            section => 'general',
+            summary =>
+                "S3 repository secret access key.",
+            description =>
+                "AWS secret key used to access this bucket."
+        },
+
+        # REPO-S3-REGION Option Help
+        #---------------------------------------------------------------------------------------------------------------------------
+        'repo-s3-region' =>
+        {
+            section => 'general',
+            summary =>
+                "S3 repository region.",
+            description =>
+                "The AWS region where the bucket was created."
+        },
+
+        # REPO-S3-VERIFY-SSL Option Help
+        #---------------------------------------------------------------------------------------------------------------------------
+        'repo-s3-verify-ssl' =>
+        {
+            section => 'general',
+            summary =>
+                "Verify S3 server certificate.",
+            description =>
+                "Disables verification of the S3 server certificate. This should only be used for testing or other scenarios " .
+                    "where a certificate has been self-signed."
+        },
+
         # REPO-TYPE Option Help
         #---------------------------------------------------------------------------------------------------------------------------
         'repo-type' =>
@@ -619,7 +700,8 @@ my $oConfigHelpData =
                 "The following repository types are supported:\n" .
                 "\n" .
                 "* cifs - Like posix, but disables links and directory fsyncs\n" .
-                "* posix - Posix-compliant file systems"
+                "* posix - Posix-compliant file systems\n" .
+                "* s3 - AWS Simple Storage Service"
         },
 
         # RESUME Option Help
@@ -833,6 +915,13 @@ my $oConfigHelpData =
                 'neutral-umask' => 'section',
                 'protocol-timeout' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'stanza' => 'default'
             }
@@ -875,6 +964,13 @@ my $oConfigHelpData =
                 'process-max' => 'section',
                 'protocol-timeout' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'spool-path' => 'section',
                 'stanza' => 'default'
@@ -946,6 +1042,13 @@ my $oConfigHelpData =
                 'process-max' => 'section',
                 'protocol-timeout' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'resume' => 'section',
                 'retention-archive' => 'section',
@@ -1023,6 +1126,13 @@ my $oConfigHelpData =
                 'online' => 'default',
                 'protocol-timeout' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'stanza' => 'default'
             }
@@ -1054,6 +1164,13 @@ my $oConfigHelpData =
                 'log-path' => 'section',
                 'log-timestamp' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'retention-archive' => 'section',
                 'retention-archive-type' => 'section',
@@ -1124,6 +1241,13 @@ my $oConfigHelpData =
 
                 'protocol-timeout' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'stanza' => 'default'
             }
@@ -1188,6 +1312,13 @@ my $oConfigHelpData =
                 'protocol-timeout' => 'section',
                 'recovery-option' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
 
                 # SET Option Help
@@ -1330,6 +1461,13 @@ my $oConfigHelpData =
                 'online' => 'default',
                 'protocol-timeout' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'stanza' => 'default'
             }
@@ -1373,6 +1511,13 @@ my $oConfigHelpData =
                 'online' => 'default',
                 'protocol-timeout' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'stanza' => 'default'
             }
@@ -1407,6 +1552,13 @@ my $oConfigHelpData =
                 'log-path' => 'section',
                 'log-timestamp' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'stanza' => 'default'
             }
@@ -1457,6 +1609,13 @@ my $oConfigHelpData =
                 'log-path' => 'section',
                 'log-timestamp' => 'section',
                 'repo-path' => 'section',
+                'repo-s3-bucket' => 'section',
+                'repo-s3-endpoint' => 'section',
+                'repo-s3-host' => 'section',
+                'repo-s3-key' => 'section',
+                'repo-s3-key-secret' => 'section',
+                'repo-s3-region' => 'section',
+                'repo-s3-verify-ssl' => 'section',
                 'repo-type' => 'section',
                 'stanza' => 'default'
             }

lib/pgBackRest/Protocol/Storage/Helper.pm

@@ -184,7 +184,17 @@ sub storageRepo
             # Create the driver
             my $oDriver;
 
-            if (optionTest(OPTION_REPO_TYPE, REPO_TYPE_CIFS))
+            if (optionTest(OPTION_REPO_TYPE, REPO_TYPE_S3))
+            {
+                require pgBackRest::Storage::S3::Driver;
+
+                $oDriver = new pgBackRest::Storage::S3::Driver(
+                    optionGet(OPTION_REPO_S3_BUCKET), optionGet(OPTION_REPO_S3_ENDPOINT), optionGet(OPTION_REPO_S3_REGION),
+                    optionGet(OPTION_REPO_S3_KEY), optionGet(OPTION_REPO_S3_KEY_SECRET),
+                    {strHost => optionGet(OPTION_REPO_S3_HOST, false), bVerifySsl => optionGet(OPTION_REPO_S3_VERIFY_SSL, false)},
+                        lBufferMax => optionGet(OPTION_BUFFER_SIZE));
+            }
+            elsif (optionTest(OPTION_REPO_TYPE, REPO_TYPE_CIFS))
             {
                 require pgBackRest::Storage::Cifs::Driver;
 

lib/pgBackRest/Storage/S3/Auth.pm

@@ -0,0 +1,264 @@
+####################################################################################################################################
+# S3 Authentication
+#
+# Contains the functions required to do S3 authentication.  It's a complicated topic and too much to cover here, but there is
+# excellent documentation at http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html.
+####################################################################################################################################
+package pgBackRest::Storage::S3::Auth;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Digest::SHA qw(hmac_sha256 hmac_sha256_hex sha256_hex);
+use Exporter qw(import);
+    our @EXPORT = qw();
+use POSIX qw(strftime);
+
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# Constants
+####################################################################################################################################
+use constant S3                                                     => 's3';
+use constant AWS4                                                   => 'AWS4';
+use constant AWS4_REQUEST                                           => 'aws4_request';
+use constant AWS4_HMAC_SHA256                                       => 'AWS4-HMAC-SHA256';
+
+use constant S3_HEADER_AUTHORIZATION                                => 'authorization';
+    push @EXPORT, qw(S3_HEADER_AUTHORIZATION);
+use constant S3_HEADER_DATE                                         => 'x-amz-date';
+    push @EXPORT, qw(S3_HEADER_DATE);
+use constant S3_HEADER_CONTENT_SHA256                               => 'x-amz-content-sha256';
+    push @EXPORT, qw(S3_HEADER_CONTENT_SHA256);
+use constant S3_HEADER_HOST                                         => 'host';
+    push @EXPORT, qw(S3_HEADER_HOST);
+
+use constant PAYLOAD_DEFAULT_HASH                                   => sha256_hex('');
+    push @EXPORT, qw(PAYLOAD_DEFAULT_HASH);
+
+####################################################################################################################################
+# s3DateTime - format date/time for authentication
+####################################################################################################################################
+sub s3DateTime
+{
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $lTime,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '::s3DateTime', \@_,
+            {name => 'lTime', default => time(), trace => true},
+        );
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'strDateTime', value => strftime("%Y%m%dT%k%M%SZ", gmtime($lTime)), trace => true}
+    );
+}
+
+push @EXPORT, qw(s3DateTime);
+
+####################################################################################################################################
+# s3CanonicalRequest - strictly formatted version of the HTTP request used for signing
+####################################################################################################################################
+sub s3CanonicalRequest
+{
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strVerb,
+        $strUri,
+        $strQuery,
+        $hHeader,
+        $strPayloadHash,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '::s3CanonicalRequest', \@_,
+            {name => 'strVerb', trace => true},
+            {name => 'strUri', trace => true},
+            {name => 'strQuery', trace => true},
+            {name => 'hHeader', trace => true},
+            {name => 'strPayloadHash', trace => true},
+        );
+
+    # Create the canonical request
+    my $strCanonicalRequest =
+        "${strVerb}\n${strUri}\n${strQuery}\n";
+    my $strSignedHeaders;
+
+    foreach my $strHeader (sort(keys(%{$hHeader})))
+    {
+        if (lc($strHeader) ne $strHeader)
+        {
+            confess &log(ASSERT, "header '${strHeader}' must be lower case");
+        }
+
+        $strCanonicalRequest .= $strHeader . ":$hHeader->{$strHeader}\n";
+        $strSignedHeaders .= (defined($strSignedHeaders) ? qw(;) : '') . lc($strHeader);
+    }
+
+    $strCanonicalRequest .= "\n${strSignedHeaders}\n${strPayloadHash}";
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'strCanonicalRequest', value => $strCanonicalRequest, trace => true},
+        {name => 'strSignedHeaders', value => $strSignedHeaders, trace => true},
+    );
+}
+
+push @EXPORT, qw(s3CanonicalRequest);
+
+####################################################################################################################################
+# s3SigningKey - signing keys last for seven days, but we'll regenerate every day because it doesn't seem too burdensome
+####################################################################################################################################
+my $hSigningKeyCache;                                               # Cache signing keys rather than regenerating them every time
+
+sub s3SigningKey
+{
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strDate,
+        $strRegion,
+        $strSecretAccessKey,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '::s3SigningKey', \@_,
+            {name => 'strDate', trace => true},
+            {name => 'strRegion', trace => true},
+            {name => 'strSecretAccessKey', trace => true},
+        );
+
+    # Check for signing key in cache
+    my $strSigningKey = $hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey};
+
+    # If not found then generate it
+    if (!defined($strSigningKey))
+    {
+        my $strDateKey = hmac_sha256($strDate, AWS4 . $strSecretAccessKey);
+        my $strRegionKey = hmac_sha256($strRegion, $strDateKey);
+        my $strServiceKey = hmac_sha256(S3, $strRegionKey);
+        $strSigningKey = hmac_sha256(AWS4_REQUEST, $strServiceKey);
+
+        # Cache the signing key
+        $hSigningKeyCache->{$strDate}{$strRegion}{$strSecretAccessKey} = $strSigningKey;
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'strSigningKey', value => $strSigningKey, trace => true}
+    );
+}
+
+push @EXPORT, qw(s3SigningKey);
+
+####################################################################################################################################
+# s3StringToSign - string that will be signed by the signing key for authentication
+####################################################################################################################################
+sub s3StringToSign
+{
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strDateTime,
+        $strRegion,
+        $strCanonicalRequestHash,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '::s3StringToSign', \@_,
+            {name => 'strDateTime', trace => true},
+            {name => 'strRegion', trace => true},
+            {name => 'strCanonicalRequestHash', trace => true},
+        );
+
+    my $strStringToSign =
+        AWS4_HMAC_SHA256 . "\n${strDateTime}\n" . substr($strDateTime, 0, 8) . "/${strRegion}/" . S3 . '/' . AWS4_REQUEST . "\n" .
+        $strCanonicalRequestHash;
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'strStringToSign', value => $strStringToSign, trace => true}
+    );
+}
+
+push @EXPORT, qw(s3StringToSign);
+
+####################################################################################################################################
+# s3AuthorizationHeader - authorization string that will be used in the HTTP "authorization" header
+####################################################################################################################################
+sub s3AuthorizationHeader
+{
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strRegion,
+        $strHost,
+        $strVerb,
+        $strUri,
+        $strQuery,
+        $strDateTime,
+        $hHeader,
+        $strAccessKeyId,
+        $strSecretAccessKey,
+        $strPayloadHash,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '::s3AuthorizationHeader', \@_,
+            {name => 'strRegion', trace => true},
+            {name => 'strHost', trace => true},
+            {name => 'strVerb', trace => true},
+            {name => 'strUri', trace => true},
+            {name => 'strQuery', trace => true},
+            {name => 'strDateTime', trace => true},
+            {name => 'hHeader', required => false, trace => true},
+            {name => 'strAccessKeyId', trace => true},
+            {name => 'strSecretAccessKey', trace => true},
+            {name => 'strPayloadHash', trace => true},
+        );
+
+    # Add s3 required headers
+    $hHeader->{&S3_HEADER_HOST} = $strHost;
+    $hHeader->{&S3_HEADER_CONTENT_SHA256} = $strPayloadHash;
+    $hHeader->{&S3_HEADER_DATE} = $strDateTime;
+
+    # Create authorization string
+    my ($strCanonicalRequest, $strSignedHeaders) = s3CanonicalRequest($strVerb, $strUri, $strQuery, $hHeader, $strPayloadHash);
+
+    $hHeader->{&S3_HEADER_AUTHORIZATION} =
+        AWS4_HMAC_SHA256 . " Credential=${strAccessKeyId}/" . substr($strDateTime, 0, 8) . "/${strRegion}/" . S3 . qw(/) .
+            AWS4_REQUEST . ",SignedHeaders=${strSignedHeaders},Signature=" . hmac_sha256_hex(s3StringToSign(
+                $strDateTime, $strRegion, sha256_hex($strCanonicalRequest)),
+                s3SigningKey(substr($strDateTime, 0, 8), $strRegion, $strSecretAccessKey));
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'hHeader', value => $hHeader, trace => true}
+    );
+}
+
+push @EXPORT, qw(s3AuthorizationHeader);
+
+1;

lib/pgBackRest/Storage/S3/Driver.pm

@@ -0,0 +1,499 @@
+####################################################################################################################################
+# S3 Storage Driver
+####################################################################################################################################
+package pgBackRest::Storage::S3::Driver;
+use parent 'pgBackRest::Storage::S3::Request';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Exporter qw(import);
+    our @EXPORT = qw();
+use Digest::MD5 qw(md5_base64);
+use File::Basename qw(basename dirname);
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+use pgBackRest::Common::Xml;
+use pgBackRest::Storage::S3::FileRead;
+use pgBackRest::Storage::S3::FileWrite;
+use pgBackRest::Storage::S3::Request;
+use pgBackRest::Storage::S3::Info;
+
+####################################################################################################################################
+# Package name constant
+####################################################################################################################################
+use constant STORAGE_S3_DRIVER                                      => __PACKAGE__;
+    push @EXPORT, qw(STORAGE_S3_DRIVER);
+
+####################################################################################################################################
+# Query constants
+####################################################################################################################################
+use constant S3_QUERY_CONTINUATION_TOKEN                            => 'continuation-token';
+use constant S3_QUERY_DELIMITER                                     => 'delimiter';
+use constant S3_QUERY_LIST_TYPE                                     => 'list-type';
+use constant S3_QUERY_PREFIX                                        => 'prefix';
+
+####################################################################################################################################
+# Batch maximum size
+####################################################################################################################################
+use constant S3_BATCH_MAX                                           => 1000;
+
+####################################################################################################################################
+# openWrite - open a file for write
+####################################################################################################################################
+sub openWrite
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strFile,
+    ) =
+        logDebugParam
+    (
+        __PACKAGE__ . '->openWrite', \@_,
+        {name => 'strFile', trace => true},
+    );
+
+    my $oFileIO = new pgBackRest::Storage::S3::FileWrite($self, $strFile);
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'oFileIO', value => $oFileIO, trace => true},
+    );
+}
+
+####################################################################################################################################
+# openRead
+####################################################################################################################################
+sub openRead
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strFile,
+        $bIgnoreMissing,
+    ) =
+        logDebugParam
+    (
+        __PACKAGE__ . '->openRead', \@_,
+        {name => 'strFile', trace => true},
+        {name => 'bIgnoreMissing', optional => true, default => false, trace => true},
+    );
+
+    my $oFileIO = new pgBackRest::Storage::S3::FileRead($self, $strFile, {bIgnoreMissing => $bIgnoreMissing});
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'oFileIO', value => $oFileIO, trace => true},
+    );
+}
+
+####################################################################################################################################
+# manifest
+####################################################################################################################################
+sub manifest
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strPath,
+        $bRecurse,
+        $bPath,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->manifest', \@_,
+            {name => 'strPath', trace => true},
+            # Optional parameters not part of the driver spec
+            {name => 'bRecurse', optional => true, default => true, trace => true},
+            {name => 'bPath', optional => true, default => true, trace => true},
+        );
+
+    # Determine the prefix (this is the search path within the bucket)
+    my $strPrefix = $strPath eq qw{/} ? undef : substr($strPath, 1) . ($bPath ? qw{/} : '');
+
+    # A delimiter must be used if recursion is not desired
+    my $strDelimiter = $bRecurse ? undef : '/';
+
+    # Hash to hold the manifest
+    my $hManifest = {};
+
+    # Continuation token - returned from requests where there is more data to be fetched
+    my $strContinuationToken;
+
+    do
+    {
+        # Get the file list
+        my $oResponse = $self->request(
+            HTTP_VERB_GET, {hQuery =>
+            {&S3_QUERY_LIST_TYPE => 2, &S3_QUERY_PREFIX => $strPrefix, &S3_QUERY_DELIMITER => $strDelimiter,
+                &S3_QUERY_CONTINUATION_TOKEN => $strContinuationToken}, strResponseType => S3_RESPONSE_TYPE_XML});
+
+        # Modify the prefix for file searches so the filename is not stripped off
+        if (defined($strPrefix) && !$bPath)
+        {
+            # If there are no paths in the prefix then undef it
+            if (index($strPrefix, qw{/}) == -1)
+            {
+                undef($strPrefix);
+            }
+            else
+            {
+                $strPrefix = dirname($strPrefix) . qw{/};
+            }
+        }
+
+        # Store files
+        foreach my $oFile (xmlTagChildren($oResponse, "Contents"))
+        {
+            my $strName = xmlTagText($oFile, "Key");
+
+            # Strip off prefix
+            if (defined($strPrefix))
+            {
+                $strName = substr($strName, length($strPrefix));
+            }
+
+            $hManifest->{$strName}->{type} = 'f';
+            $hManifest->{$strName}->{size} = xmlTagText($oFile, "Size");
+
+            # Generate paths from the name if recursing
+            if ($bRecurse)
+            {
+                my @stryName = split(qw{/}, $strName);
+
+                if (@stryName > 1)
+                {
+                    $strName = undef;
+
+                    for (my $iIndex = 0; $iIndex < @stryName - 1; $iIndex++)
+                    {
+                        $strName .= (defined($strName) ? qw{/} : '') . $stryName[$iIndex];
+                        $hManifest->{$strName}->{type} = 'd';
+                    }
+                }
+            }
+        }
+
+        # Store directories
+        if ($bPath && !$bRecurse)
+        {
+            foreach my $oPath (xmlTagChildren($oResponse, "CommonPrefixes"))
+            {
+                my $strName = xmlTagText($oPath, "Prefix");
+
+                # Strip off prefix
+                if (defined($strPrefix))
+                {
+                    $strName = substr($strName, length($strPrefix));
+                }
+
+                # Strip off final /
+                $strName = substr($strName, 0, length($strName) - 1);
+
+                $hManifest->{$strName}->{type} = 'd';
+            }
+        }
+
+        $strContinuationToken = xmlTagText($oResponse, "NextContinuationToken", false);
+    }
+    while (defined($strContinuationToken));
+
+    # Add . for the initial path (this is just for compatibility with filesystems that have directories)
+    if ($bPath)
+    {
+        $hManifest->{qw{.}}->{type} = 'd';
+    }
+
+    # use Data::Dumper; &log(WARN, 'MANIFEST' . Dumper($hManifest));
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'hManifest', value => $hManifest, trace => true}
+    );
+}
+
+####################################################################################################################################
+# list - list a directory
+####################################################################################################################################
+sub list
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strPath,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->list', \@_,
+            {name => 'strPath', trace => true},
+        );
+
+    # Get list using manifest function
+    my @stryFileList = grep(!/^\.$/i, keys(%{$self->manifest($strPath, {bRecurse => false})}));
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'stryFileList', value => \@stryFileList, ref => true, trace => true}
+    );
+}
+
+####################################################################################################################################
+# pathCreate - directories do no exist in s3 so this is a noop
+####################################################################################################################################
+sub pathCreate
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strPath,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->pathCreate', \@_,
+            {name => 'strPath', trace => true},
+        );
+
+    # Return from function and log return values if any
+    return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# pathSync - directories do not exist in s3 so this is a noop
+####################################################################################################################################
+sub pathSync
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strPath,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->pathSync', \@_,
+            {name => 'strPath', trace => true},
+        );
+
+    # Return from function and log return values if any
+    return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# exists - check if a file exists
+####################################################################################################################################
+sub exists
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strFile,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->exists', \@_,
+            {name => 'strFile', trace => true},
+        );
+
+    # Does the path/file exist?
+    my $bExists = defined($self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)}) ? true : false;
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'bExists', value => $bExists, trace => true}
+    );
+}
+
+####################################################################################################################################
+# pathExists
+####################################################################################################################################
+sub pathExists
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strPath,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->pathExists', \@_,
+            {name => 'strPath', trace => true},
+        );
+
+    my $bExists = true;
+
+    # Only check if path <> /
+    if ($strPath ne qw{/})
+    {
+        # Does the path exist?
+        my $rhInfo = $self->manifest(dirname($strPath), {bRecurse => false, bPath => true})->{basename($strPath)};
+        $bExists = defined($rhInfo) && $rhInfo->{type} eq 'd' ? true : false;
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'bExists', value => $bExists, trace => true}
+    );
+}
+
+####################################################################################################################################
+# info - get information about a file
+####################################################################################################################################
+sub info
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strFile,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->info', \@_,
+            {name => 'strFile', trace => true},
+        );
+
+    # Get the file
+    my $rhFile = $self->manifest($strFile, {bRecurse => false, bPath => false})->{basename($strFile)};
+
+    if (!defined($rhFile))
+    {
+        confess &log(ERROR, "unable to get info for missing file ${strFile}", ERROR_FILE_MISSING);
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'oInfo', value => new pgBackRest::Storage::S3::Info($rhFile->{size}), trace => true}
+    );
+}
+
+####################################################################################################################################
+# remove
+####################################################################################################################################
+sub remove
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $rstryFile,
+        $bRecurse,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->remove', \@_,
+            {name => 'rstryFile', trace => true},
+            {name => 'bRecurse', optional => true, default => false, trace => true},
+        );
+
+    # Remove a tree
+    if ($bRecurse)
+    {
+        my $rhManifest = $self->manifest($rstryFile);
+        my @stryRemoveFile;
+
+        # Iterate all files in the manifest
+        foreach my $strFile (sort({$b cmp $a} keys(%{$rhManifest})))
+        {
+            next if $rhManifest->{$strFile}->{type} eq 'd';
+            push(@stryRemoveFile, "${rstryFile}/${strFile}");
+        }
+
+        # Remove files
+        if (@stryRemoveFile > 0)
+        {
+            $self->remove(\@stryRemoveFile);
+        }
+    }
+    # Only remove the specified file
+    else
+    {
+        # If stryFile is a scalar, convert to an array
+        my $rstryFileAll = ref($rstryFile) ? $rstryFile : [$rstryFile];
+
+        do
+        {
+            my $strFile = shift(@{$rstryFileAll});
+            my $iTotal = 0;
+            my $strXml = XML_HEADER . '<Delete><Quiet>true</Quiet>';
+
+            while (defined($strFile))
+            {
+                $iTotal++;
+                $strXml .= '<Object><Key>' . substr($strFile, 1) . '</Key></Object>';
+
+                $strFile = $iTotal < 2 ? shift(@{$rstryFileAll}) : undef;
+            }
+
+            $strXml .= '</Delete>';
+
+            my $hHeader = {'content-md5' => md5_base64($strXml) . '=='};
+
+            # Delete a file
+            my $oResponse = $self->request(
+                HTTP_VERB_POST,
+                {hQuery => 'delete=', rstrBody => \$strXml, hHeader => $hHeader, strResponseType => S3_RESPONSE_TYPE_XML});
+        }
+        while (@{$rstryFileAll} > 0);
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'bResult', value => true, trace => true}
+    );
+}
+
+
+####################################################################################################################################
+# Getters
+####################################################################################################################################
+sub capability {false}
+sub className {STORAGE_S3_DRIVER}
+
+1;

lib/pgBackRest/Storage/S3/FileRead.pm

@@ -0,0 +1,66 @@
+####################################################################################################################################
+# S3 File Read
+####################################################################################################################################
+package pgBackRest::Storage::S3::FileRead;
+use parent 'pgBackRest::Common::Io::Base';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Digest::MD5 qw(md5_base64);
+use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);
+use File::Basename qw(dirname);
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+use pgBackRest::Common::Xml;
+use pgBackRest::Storage::Base;
+use pgBackRest::Storage::S3::Request;
+
+####################################################################################################################################
+# CONSTRUCTOR
+####################################################################################################################################
+our @ISA = ();                                                      ## no critic (ClassHierarchies::ProhibitExplicitISA)
+
+sub new
+{
+    my $class = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $oDriver,
+        $strName,
+        $bIgnoreMissing,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->new', \@_,
+            {name => 'oDriver', trace => true},
+            {name => 'strName', trace => true},
+            {name => 'bIgnoreMissing', optional => true, default => false, trace => true},
+        );
+
+    # Open file
+    my $self = $oDriver->request(
+        HTTP_VERB_GET, {strUri => $strName, strResponseType => S3_RESPONSE_TYPE_IO, bIgnoreMissing => $bIgnoreMissing});
+
+    # Bless with new class if file exists
+    if (defined($self))
+    {
+        @ISA = $self->isA();                                        ## no critic (ClassHierarchies::ProhibitExplicitISA)
+        bless $self, $class;
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'self', value => $self, trace => true}
+    );
+}
+
+1;

lib/pgBackRest/Storage/S3/FileWrite.pm

@@ -0,0 +1,194 @@
+####################################################################################################################################
+# S3 File Write
+####################################################################################################################################
+package pgBackRest::Storage::S3::FileWrite;
+use parent 'pgBackRest::Common::Io::Base';
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Digest::MD5 qw(md5_base64);
+use Fcntl qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);
+use File::Basename qw(dirname);
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+use pgBackRest::Common::Xml;
+use pgBackRest::Storage::Base;
+use pgBackRest::Storage::S3::Request;
+
+####################################################################################################################################
+# Constants
+####################################################################################################################################
+use constant S3_BUFFER_MAX                                          => 16777216;
+
+####################################################################################################################################
+# CONSTRUCTOR
+####################################################################################################################################
+sub new
+{
+    my $class = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $oDriver,
+        $strName,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->new', \@_,
+            {name => 'oDriver', trace => true},
+            {name => 'strName', trace => true},
+        );
+
+    # Create the class hash
+    my $self = $class->SUPER::new("'${strName}'");
+    bless $self, $class;
+
+    # Set variables
+    $self->{oDriver} = $oDriver;
+    $self->{strName} = $strName;
+
+    # Start with an empty buffer
+    $self->{rtBuffer} = '';
+
+    # Has anything been written?
+    $self->{bWritten} = false;
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'self', value => $self, trace => true}
+    );
+}
+
+####################################################################################################################################
+# open - open the file
+####################################################################################################################################
+sub open
+{
+    my $self = shift;
+
+    # Request an upload id
+    my $oResponse = $self->{oDriver}->request(
+        HTTP_VERB_POST, {strUri => $self->{strName}, hQuery => 'uploads=', strResponseType => S3_RESPONSE_TYPE_XML});
+
+    $self->{strUploadId} = xmlTagText($oResponse, 'UploadId');
+
+    # Intialize the multi-part array
+    $self->{rstryMultiPart} = [];
+}
+
+####################################################################################################################################
+# write - write data to a file
+####################################################################################################################################
+sub write
+{
+    my $self = shift;
+    my $rtBuffer = shift;
+
+    # Note that write has been called
+    $self->{bWritten} = true;
+
+    if (defined($rtBuffer))
+    {
+        $self->{rtBuffer} .= $$rtBuffer;
+
+        # Wait until buffer is at least max before writing to avoid writing smaller files multi-part
+        if (length($self->{rtBuffer}) >= S3_BUFFER_MAX)
+        {
+            $self->flush();
+        }
+
+        return length($$rtBuffer);
+    }
+
+    return 0;
+}
+
+####################################################################################################################################
+# flush - flush whatever is in the buffer out
+####################################################################################################################################
+sub flush
+{
+    my $self = shift;
+
+    # Open file if it is not open already
+    $self->open() if !$self->opened();
+
+    # Put a file
+    $self->{oDriver}->request(
+        HTTP_VERB_PUT,
+        {strUri => $self->{strName},
+            hQuery => {'partNumber' => @{$self->{rstryMultiPart}} + 1, 'uploadId' => $self->{strUploadId}},
+            rstrBody => \$self->{rtBuffer}, hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}});
+
+    # Store the returned etag
+    push(@{$self->{rstryMultiPart}}, $self->{oDriver}->{hResponseHeader}{&S3_HEADER_ETAG});
+
+    # Clear the buffer
+    $self->{rtBuffer} = '';
+}
+
+####################################################################################################################################
+# close - close the file
+####################################################################################################################################
+sub close
+{
+    my $self = shift;
+
+    # Only close if something was written
+    if ($self->{bWritten})
+    {
+        # Make sure close does not run again
+        $self->{bWritten} = false;
+
+        # If the file is open then multipart transfer has already started and must be completed
+        if ($self->opened())
+        {
+            # flush out whatever is in the buffer
+            $self->flush();
+
+            my $strXml = XML_HEADER . '<CompleteMultipartUpload>';
+            my $iPartNo = 0;
+
+            foreach my $strETag (@{$self->{rstryMultiPart}})
+            {
+                $iPartNo++;
+
+                $strXml .= "<Part><PartNumber>${iPartNo}</PartNumber><ETag>${strETag}</ETag></Part>";
+            }
+
+            $strXml .= '</CompleteMultipartUpload>';
+
+            # Finalize file
+            my $oResponse = $self->{oDriver}->request(
+                HTTP_VERB_POST,
+                {strUri => $self->{strName}, hQuery => {'uploadId' => $self->{strUploadId}},
+                    rstrBody => \$strXml, hHeader => {'content-md5' => md5_base64($strXml) . '=='},
+                    strResponseType => S3_RESPONSE_TYPE_XML});
+        }
+        # Else the file can be transmitted in one block
+        else
+        {
+            $self->{oDriver}->request(
+                HTTP_VERB_PUT,
+                {strUri => $self->{strName}, rstrBody => \$self->{rtBuffer},
+                    hHeader => {'content-md5' => md5_base64($self->{rtBuffer}) . '=='}});
+        }
+    }
+
+    return true;
+}
+
+####################################################################################################################################
+# Getters
+####################################################################################################################################
+sub opened {defined(shift->{strUploadId})}
+
+1;

lib/pgBackRest/Storage/S3/Info.pm

@@ -0,0 +1,48 @@
+####################################################################################################################################
+# S3 File Info
+####################################################################################################################################
+package pgBackRest::Storage::S3::Info;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+sub new
+{
+    my $class = shift;
+
+    # Create the class hash
+    my $self = {};
+    bless $self, $class;
+
+    # Assign function parameters, defaults, and log debug info
+    (
+        my $strOperation,
+        $self->{lSize},
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->new', \@_,
+            {name => 'lSize'},
+        );
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'self', value => $self}
+    );
+}
+
+####################################################################################################################################
+# Getters
+####################################################################################################################################
+sub size {shift->{lSize}}
+
+1;

lib/pgBackRest/Storage/S3/Request.pm

@@ -0,0 +1,202 @@
+####################################################################################################################################
+# S3 Request
+####################################################################################################################################
+package pgBackRest::Storage::S3::Request;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Digest::SHA qw(hmac_sha256 hmac_sha256_hex sha256_hex);
+use Exporter qw(import);
+    our @EXPORT = qw();
+use IO::Socket::SSL;
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Http::Client;
+use pgBackRest::Common::Http::Common;
+use pgBackRest::Common::Io::Base;
+use pgBackRest::Common::Log;
+use pgBackRest::Common::String;
+use pgBackRest::Common::Xml;
+use pgBackRest::Storage::S3::Auth;
+
+####################################################################################################################################
+# Constants
+####################################################################################################################################
+use constant HTTP_VERB_GET                                          => 'GET';
+    push @EXPORT, qw(HTTP_VERB_GET);
+use constant HTTP_VERB_POST                                         => 'POST';
+    push @EXPORT, qw(HTTP_VERB_POST);
+use constant HTTP_VERB_PUT                                          => 'PUT';
+    push @EXPORT, qw(HTTP_VERB_PUT);
+
+use constant S3_HEADER_CONTENT_LENGTH                               => 'content-length';
+    push @EXPORT, qw(S3_HEADER_CONTENT_LENGTH);
+use constant S3_HEADER_TRANSFER_ENCODING                            => 'transfer-encoding';
+    push @EXPORT, qw(S3_HEADER_TRANSFER_ENCODING);
+use constant S3_HEADER_ETAG                                         => 'etag';
+    push @EXPORT, qw(S3_HEADER_ETAG);
+
+use constant S3_RESPONSE_TYPE_IO                                    => 'io';
+    push @EXPORT, qw(S3_RESPONSE_TYPE_IO);
+use constant S3_RESPONSE_TYPE_NONE                                  => 'none';
+    push @EXPORT, qw(S3_RESPONSE_TYPE_NONE);
+use constant S3_RESPONSE_TYPE_XML                                   => 'xml';
+    push @EXPORT, qw(S3_RESPONSE_TYPE_XML);
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+sub new
+{
+    my $class = shift;
+
+    # Create the class hash
+    my $self = {};
+    bless $self, $class;
+
+    # Assign function parameters, defaults, and log debug info
+    (
+        my $strOperation,
+        $self->{strBucket},
+        $self->{strEndPoint},
+        $self->{strRegion},
+        $self->{strAccessKeyId},
+        $self->{strSecretAccessKey},
+        $self->{strHost},
+        $self->{bVerifySsl},
+        $self->{lBufferMax},
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->new', \@_,
+            {name => 'strBucket', trace => true},
+            {name => 'strEndPoint', trace => true},
+            {name => 'strRegion', trace => true},
+            {name => 'strAccessKeyId', trace => true},
+            {name => 'strSecretAccessKey', trace => true},
+            {name => 'strHost', optional => true, trace => true},
+            {name => 'bVerifySsl', optional => true, default => true, trace => true},
+            {name => 'lBufferMax', optional => true, default => COMMON_IO_BUFFER_MAX, trace => true},
+        );
+
+    # If host is not set then it will be bucket + endpoint
+    $self->{strHost} = defined($self->{strHost}) ? $self->{strHost} : "$self->{strBucket}.$self->{strEndPoint}";
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'self', value => $self, trace => true}
+    );
+}
+
+####################################################################################################################################
+# request - send a request to S3
+####################################################################################################################################
+sub request
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strVerb,
+        $strUri,
+        $hQuery,
+        $hHeader,
+        $rstrBody,
+        $strResponseType,
+        $bIgnoreMissing,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->request', \@_,
+            {name => 'strVerb', trace => true},
+            {name => 'strUri', optional => true, default => '/', trace => true},
+            {name => 'hQuery', optional => true, trace => true},
+            {name => 'hHeader', optional => true, trace => true},
+            {name => 'rstrBody', optional => true, trace => true},
+            {name => 'strResponseType', optional => true, default => S3_RESPONSE_TYPE_NONE, trace => true},
+            {name => 'bIgnoreMissing', optional => true, default => false, trace => true},
+        );
+
+    # Get datetime to be used for auth requests
+    my $strDateTime = s3DateTime();
+
+    # Set content length and hash
+    $hHeader->{&S3_HEADER_CONTENT_SHA256} = defined($rstrBody) ? sha256_hex($$rstrBody) : PAYLOAD_DEFAULT_HASH;
+    $hHeader->{&S3_HEADER_CONTENT_LENGTH} = defined($rstrBody) ? length($$rstrBody) : 0;
+
+    # Generate authorization header
+    $hHeader = s3AuthorizationHeader(
+        $self->{strRegion}, "$self->{strBucket}.$self->{strEndPoint}", $strVerb, $strUri, httpQuery($hQuery), $strDateTime,
+        $hHeader, $self->{strAccessKeyId}, $self->{strSecretAccessKey}, $hHeader->{&S3_HEADER_CONTENT_SHA256});
+
+    # Send the request
+    my $oHttpClient = new pgBackRest::Common::Http::Client(
+        $self->{strHost}, $strVerb,
+        {strUri => $strUri, hQuery => $hQuery, hRequestHeader => $hHeader, rstrRequestBody => $rstrBody,
+            bVerifySsl => $self->{bVerifySsl}, lBufferMax => $self->{lBufferMax}});
+
+    # Check response code
+    my $iReponseCode = $oHttpClient->responseCode();
+    my $oResponse;
+
+    if ($iReponseCode == 200)
+    {
+        # Save the response headers locally
+        $self->{hResponseHeader} = $oHttpClient->responseHeader();
+
+        # XML response is expected
+        if ($strResponseType eq S3_RESPONSE_TYPE_XML)
+        {
+            my $rtResponseBody = $oHttpClient->responseBody();
+
+            if ($oHttpClient->contentLength() == 0 || !defined($$rtResponseBody))
+            {
+                confess &log(ERROR,
+                    "response type '${strResponseType}' was requested but content length is zero or content is missing",
+                    ERROR_PROTOCOL);
+            }
+
+            $oResponse = xmlParse($$rtResponseBody);
+        }
+        # An IO object is expected for file responses
+        elsif ($strResponseType eq S3_RESPONSE_TYPE_IO)
+        {
+            $oResponse = $oHttpClient;
+        }
+    }
+    else
+    {
+        if ($iReponseCode == 404)
+        {
+            if (!$bIgnoreMissing)
+            {
+                confess &log(ERROR, "unable to open '${strUri}': No such file or directory", ERROR_FILE_MISSING);
+            }
+        }
+        else
+        {
+            my $rstrResponseBody = $oHttpClient->responseBody();
+
+            confess &log(ERROR,
+                "S3 request error [$iReponseCode] " . $oHttpClient->responseMessage() .
+                    (defined($$rstrResponseBody) ? ":\n${$rstrResponseBody}" : ''),
+                ERROR_PROTOCOL);
+        }
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'oResponse', value => $oResponse, trace => true, ref => true}
+    );
+}
+
+1;

test/Vagrantfile

@@ -8,7 +8,7 @@ Vagrant.configure(2) do |config|
     config.vm.box_version = "20170603.0.0"
 
     # vagrant plugin install vagrant-disksize
-    # config.disksize.size = '64GB'
+    config.disksize.size = '64GB'
 
     config.vm.provider :virtualbox do |vb|
         vb.name = "pgbackrest-test"
@@ -37,12 +37,29 @@ Vagrant.configure(2) do |config|
 
         #---------------------------------------------------------------------------------------------------------------------------
         echo 'Install Perl Modules' && date
-        apt-get install -y libdbd-pg-perl libxml-checker-perl libperl-critic-perl libdevel-nytprof-perl
+        apt-get install -y libdbd-pg-perl libio-socket-ssl-perl libxml-libxml-perl libxml-checker-perl libperl-critic-perl \
+            libdevel-nytprof-perl
 
         #---------------------------------------------------------------------------------------------------------------------------
         echo 'Install Build Tools' && date
         apt-get install -y devscripts build-essential lintian git txt2man debhelper
 
+        #---------------------------------------------------------------------------------------------------------------------------
+        echo 'Install AWS CLI' && date
+        apt-get install -y python-pip
+        pip install --upgrade awscli
+
+        # Configure AWS CLI
+        sudo -i -u ubuntu aws configure set region us-east-1
+        sudo -i -u ubuntu aws configure set aws_access_key_id accessKey1
+        sudo -i -u ubuntu aws configure set aws_secret_access_key verySecretKey1
+
+        # Create test alias for AWS CLI
+        echo '' >> /home/ubuntu/.profile
+        echo '# Test alias for AWS CLI' >> /home/ubuntu/.profile
+        echo 'alias s3-test="export PYTHONWARNINGS=ignore && aws s3 --endpoint-url=https://172.17.0.2 --no-verify-ssl"' \
+            >> /home/ubuntu/.profile
+
         #---------------------------------------------------------------------------------------------------------------------------
         echo 'Build Devel::Cover' && date
         apt-get install -y libpod-coverage-perl libtest-differences-perl libhtml-parser-perl libtemplate-perl

test/expect/archive-get-001.log

@@ -1,5 +1,5 @@
-run 001 - rmt 0, cmp 0, exists 0
-================================
+run 001 - rmt 0, cmp 0, exists 0, s3 0
+======================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-get-002.log

@@ -1,5 +1,5 @@
-run 002 - rmt 0, cmp 0, exists 1
-================================
+run 002 - rmt 0, cmp 0, exists 1, s3 0
+======================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-get-003.log

@@ -1,5 +1,5 @@
-run 003 - rmt 0, cmp 1, exists 0
-================================
+run 003 - rmt 0, cmp 1, exists 0, s3 0
+======================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-get-004.log

@@ -1,5 +1,5 @@
-run 004 - rmt 0, cmp 1, exists 1
-================================
+run 004 - rmt 0, cmp 1, exists 1, s3 0
+======================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-get-005.log

@@ -1,5 +1,5 @@
-run 005 - rmt 1, cmp 0, exists 0
-================================
+run 005 - rmt 1, cmp 0, exists 0, s3 0
+======================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-get-006.log

@@ -1,5 +1,5 @@
-run 006 - rmt 1, cmp 0, exists 1
-================================
+run 006 - rmt 1, cmp 0, exists 1, s3 0
+======================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-get-007.log

@@ -1,5 +1,5 @@
-run 007 - rmt 1, cmp 1, exists 0
-================================
+run 007 - rmt 1, cmp 1, exists 0, s3 0
+======================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-get-008.log

@@ -1,5 +1,5 @@
-run 008 - rmt 1, cmp 1, exists 1
-================================
+run 008 - rmt 1, cmp 1, exists 1, s3 0
+======================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-get-009.log

@@ -0,0 +1,181 @@
+run 009 - rmt 1, cmp 1, exists 1, s3 1
+======================================
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
+P00   INFO: get WAL segment 000000010000000100000001
+P00  DEBUG:     Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
+P00  DEBUG:     Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
+P00  DEBUG:     Protocol::Helper::protocolGet: create (cached) remote protocol
+P00  DEBUG:     Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1]
+P00  DEBUG:     Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00  DEBUG:     Protocol::Storage::Remote->new(): oProtocol = [object]
+P00  DEBUG:     Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00  DEBUG:     Db->new(): iRemoteIdx = 1
+P00  DEBUG:     Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
+P00  DEBUG:     Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
+P00  DEBUG:     Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00  DEBUG:     Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
+P00  DEBUG:     Protocol::Helper::protocolGet: found cached protocol
+P00  ERROR: [055]: raised on 'backup remote' host: archive.info does not exist but is required to push/get WAL segments
+            HINT: is archive_command configured in postgresql.conf?
+            HINT: has a stanza-create been performed?
+            HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
+P00  DEBUG:     Common::Exit::exitSafe(): iExitCode = [undef], oException = [object], strSignal = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy(): bComplete = false, iRemoteIdx = [undef], strRemoteType = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup
+P00  DEBUG:     Protocol::Command::Master->close=>: iExitStatus = 0
+P00  DEBUG:     Protocol::Helper::protocolDestroy=>: iExitStatus = 0
+P00  DEBUG:     Common::Lock::lockRelease(): bFailOnNoLock = false
+P00   INFO: archive-get command end: aborted with exception [055]
+P00  DEBUG:     Common::Exit::exitSafe=>: iExitCode = 55
+
+stanza-create db - create required data for stanza (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-2] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get --cmd-ssh=/usr/bin/ssh 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
+P00   INFO: get WAL segment 000000010000000100000001
+P00  DEBUG:     Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, strSourceArchive = 000000010000000100000001
+P00  DEBUG:     Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
+P00  DEBUG:     Protocol::Helper::protocolGet: create (cached) remote protocol
+P00  DEBUG:     Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = /usr/bin/ssh, strHost = backup, strUser = [USER-1]
+P00  DEBUG:     Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = /usr/bin/ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00  DEBUG:     Protocol::Storage::Remote->new(): oProtocol = [object]
+P00  DEBUG:     Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000001, ullDbSysId = [undef]
+P00  DEBUG:     Db->new(): iRemoteIdx = 1
+P00  DEBUG:     Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
+P00  DEBUG:     Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
+P00  DEBUG:     Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00  DEBUG:     Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
+P00  DEBUG:     Protocol::Helper::protocolGet: found cached protocol
+P00  DEBUG:     Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strArchiveId = 9.4-1
+P00  DEBUG:     Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = <REPO:ARCHIVE>/9.4-1/000000010000000100000001-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz
+P00  DEBUG:     Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00  DEBUG:     Storage::Local->openWrite(): bAtomic = <false>, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+P00  DEBUG:     Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
+P00  DEBUG:     Archive::ArchiveGet->get=>: iResult = 0
+P00  DEBUG:     Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup
+P00  DEBUG:     Protocol::Command::Master->close=>: iExitStatus = 0
+P00  DEBUG:     Protocol::Helper::protocolDestroy=>: iExitStatus = 0
+P00  DEBUG:     Common::Lock::lockRelease(): bFailOnNoLock = false
+P00   INFO: archive-get command end: completed successfully
+P00  DEBUG:     Common::Exit::exitSafe=>: iExitCode = 0
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000002 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
+P00   INFO: get WAL segment 000000010000000100000002
+P00  DEBUG:     Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002, strSourceArchive = 000000010000000100000002
+P00  DEBUG:     Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
+P00  DEBUG:     Protocol::Helper::protocolGet: create (cached) remote protocol
+P00  DEBUG:     Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1]
+P00  DEBUG:     Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00  DEBUG:     Protocol::Storage::Remote->new(): oProtocol = [object]
+P00  DEBUG:     Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000002, ullDbSysId = [undef]
+P00  DEBUG:     Db->new(): iRemoteIdx = 1
+P00  DEBUG:     Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
+P00  DEBUG:     Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
+P00  DEBUG:     Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00  DEBUG:     Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
+P00  DEBUG:     Protocol::Helper::protocolGet: found cached protocol
+P00  DEBUG:     Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strArchiveId = 9.4-1
+P00  DEBUG:     Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = <REPO:ARCHIVE>/9.4-1/000000010000000100000002-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz
+P00  DEBUG:     Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00  DEBUG:     Storage::Local->openWrite(): bAtomic = <false>, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
+P00  DEBUG:     Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
+P00  DEBUG:     Archive::ArchiveGet->get=>: iResult = 0
+P00  DEBUG:     Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup
+P00  DEBUG:     Protocol::Command::Master->close=>: iExitStatus = 0
+P00  DEBUG:     Protocol::Helper::protocolDestroy=>: iExitStatus = 0
+P00  DEBUG:     Common::Lock::lockRelease(): bFailOnNoLock = false
+P00   INFO: archive-get command end: completed successfully
+P00  DEBUG:     Common::Exit::exitSafe=>: iExitCode = 0
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000003 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-get command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --stanza=db
+P00   INFO: get WAL segment 000000010000000100000003
+P00  DEBUG:     Archive::ArchiveGet->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003, strSourceArchive = 000000010000000100000003
+P00  DEBUG:     Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
+P00  DEBUG:     Protocol::Helper::protocolGet: create (cached) remote protocol
+P00  DEBUG:     Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1]
+P00  DEBUG:     Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 6, iCompressLevelNetwork = 3, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no backrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=6 --compress-level-network=3 --config=[TEST_PATH]/backup/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --protocol-timeout=60 --stanza=db --type=backup remote', strId = 'backup remote', strName = remote
+P00  DEBUG:     Protocol::Storage::Remote->new(): oProtocol = [object]
+P00  DEBUG:     Archive::Archive->getCheck(): strDbVersion = [undef], strWalFile = 000000010000000100000003, ullDbSysId = [undef]
+P00  DEBUG:     Db->new(): iRemoteIdx = 1
+P00  DEBUG:     Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
+P00  DEBUG:     Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
+P00  DEBUG:     Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00  DEBUG:     Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
+P00  DEBUG:     Protocol::Helper::protocolGet: found cached protocol
+P00  DEBUG:     Archive::Archive->getCheck=>: strArchiveFile = 000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz, strArchiveId = 9.4-1
+P00  DEBUG:     Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = <REPO:ARCHIVE>/9.4-1/000000010000000100000003-72b9da071c13957fb4ca31f05dbd5c644297c2f7.gz
+P00  DEBUG:     Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00  DEBUG:     Storage::Local->openWrite(): bAtomic = <false>, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
+P00  DEBUG:     Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
+P00  DEBUG:     Archive::ArchiveGet->get=>: iResult = 0
+P00  DEBUG:     Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup
+P00  DEBUG:     Protocol::Command::Master->close=>: iExitStatus = 0
+P00  DEBUG:     Protocol::Helper::protocolDestroy=>: iExitStatus = 0
+P00  DEBUG:     Common::Lock::lockRelease(): bFailOnNoLock = false
+P00   INFO: archive-get command end: completed successfully
+P00  DEBUG:     Common::Exit::exitSafe=>: iExitCode = 0

test/expect/archive-push-001.log

@@ -1,5 +1,5 @@
-run 001 - rmt 0, cmp 0, arc_async 0
-===================================
+run 001 - rmt 0, cmp 0, arc_async 0, s3 0
+=========================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-push-002.log

@@ -1,5 +1,5 @@
-run 002 - rmt 0, cmp 0, arc_async 1
-===================================
+run 002 - rmt 0, cmp 0, arc_async 1, s3 0
+=========================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-push-003.log

@@ -1,5 +1,5 @@
-run 003 - rmt 0, cmp 1, arc_async 0
-===================================
+run 003 - rmt 0, cmp 1, arc_async 0, s3 0
+=========================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-push-004.log

@@ -1,5 +1,5 @@
-run 004 - rmt 0, cmp 1, arc_async 1
-===================================
+run 004 - rmt 0, cmp 1, arc_async 1, s3 0
+=========================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-push-005.log

@@ -1,5 +1,5 @@
-run 005 - rmt 1, cmp 0, arc_async 0
-===================================
+run 005 - rmt 1, cmp 0, arc_async 0, s3 0
+=========================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-push-006.log

@@ -1,5 +1,5 @@
-run 006 - rmt 1, cmp 0, arc_async 1
-===================================
+run 006 - rmt 1, cmp 0, arc_async 1, s3 0
+=========================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-push-007.log

@@ -1,5 +1,5 @@
-run 007 - rmt 1, cmp 1, arc_async 0
-===================================
+run 007 - rmt 1, cmp 1, arc_async 0, s3 0
+=========================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-push-008.log

@@ -1,5 +1,5 @@
-run 008 - rmt 1, cmp 1, arc_async 1
-===================================
+run 008 - rmt 1, cmp 1, arc_async 1, s3 0
+=========================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24
 ------------------------------------------------------------------------------------------------------------------------------------

test/expect/archive-push-009.log

@@ -0,0 +1,263 @@
+run 009 - rmt 1, cmp 1, arc_async 1, s3 1
+=========================================
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 --archive-max-mb=24
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --archive-max-mb=24 --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   WARN: 'archive-max-mb' is no longer not longer valid, use 'archive-queue-max' instead
+P00  ERROR: [055]: raised on 'local-1' host: raised on 'backup remote' host: archive.info does not exist but is required to push/get WAL segments
+            HINT: is archive_command configured in postgresql.conf?
+            HINT: has a stanza-create been performed?
+            HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
+P00   INFO: archive-push command end: aborted with exception [055]
+
+stanza-create db - create required data for stanza (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-2] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push --cmd-ssh=/usr/bin/ssh [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000001 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment version 9.4 does not match archive version 8.0
+            HINT: are you archiving to the correct stanza?
+P00   INFO: archive-push command end: aborted with exception [044]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment system-id 6353949018581704918 does not match archive system-id 5000900090001855000
+            HINT: are you archiving to the correct stanza?
+P00   INFO: archive-push command end: aborted with exception [044]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   WARN: WAL segment 000000010000000100000001 already exists in the archive with the same checksum
+            HINT: this is valid in some recovery scenarios but may also indicate a problem.
+P00   INFO: pushed WAL segment 000000010000000100000001 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000001 already exists in the archive
+P00   INFO: archive-push command end: aborted with exception [045]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001.partial
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000001.partial asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001.partial
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   WARN: WAL segment 000000010000000100000001.partial already exists in the archive with the same checksum
+            HINT: this is valid in some recovery scenarios but may also indicate a problem.
+P00   INFO: pushed WAL segment 000000010000000100000001.partial asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001.partial
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000001.partial already exists in the archive
+P00   INFO: archive-push command end: aborted with exception [045]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000002 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000003 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000004
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000004 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push --cmd-ssh=/usr/bin/ssh [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000005 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment version 9.4 does not match archive version 8.0
+            HINT: are you archiving to the correct stanza?
+P00   INFO: archive-push command end: aborted with exception [044]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment system-id 6353949018581704918 does not match archive system-id 5000900090001855000
+            HINT: are you archiving to the correct stanza?
+P00   INFO: archive-push command end: aborted with exception [044]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   WARN: WAL segment 000000010000000100000005 already exists in the archive with the same checksum
+            HINT: this is valid in some recovery scenarios but may also indicate a problem.
+P00   INFO: pushed WAL segment 000000010000000100000005 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000005 already exists in the archive
+P00   INFO: archive-push command end: aborted with exception [045]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005.partial
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000005.partial asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005.partial
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   WARN: WAL segment 000000010000000100000005.partial already exists in the archive with the same checksum
+            HINT: this is valid in some recovery scenarios but may also indicate a problem.
+P00   INFO: pushed WAL segment 000000010000000100000005.partial asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005.partial
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000005.partial already exists in the archive
+P00   INFO: archive-push command end: aborted with exception [045]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000006
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000006 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000007
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000007 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000008
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000008 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push --cmd-ssh=/usr/bin/ssh [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --cmd-ssh=/usr/bin/ssh --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000009 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment version 9.4 does not match archive version 8.0
+            HINT: are you archiving to the correct stanza?
+P00   INFO: archive-push command end: aborted with exception [044]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [044]: raised on 'local-1' host: raised on 'backup remote' host: WAL segment system-id 6353949018581704918 does not match archive system-id 5000900090001855000
+            HINT: are you archiving to the correct stanza?
+P00   INFO: archive-push command end: aborted with exception [044]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   WARN: WAL segment 000000010000000100000009 already exists in the archive with the same checksum
+            HINT: this is valid in some recovery scenarios but may also indicate a problem.
+P00   INFO: pushed WAL segment 000000010000000100000009 asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000009 already exists in the archive
+P00   INFO: archive-push command end: aborted with exception [045]
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009.partial
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: pushed WAL segment 000000010000000100000009.partial asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009.partial
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   WARN: WAL segment 000000010000000100000009.partial already exists in the archive with the same checksum
+            HINT: this is valid in some recovery scenarios but may also indicate a problem.
+P00   INFO: pushed WAL segment 000000010000000100000009.partial asynchronously
+P00   INFO: archive-push command end: completed successfully
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000009.partial
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --archive-async --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-1] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00  ERROR: [045]: raised on 'local-1' host: WAL segment 000000010000000100000009.partial already exists in the archive
+P00   INFO: archive-push command end: aborted with exception [045]
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}

test/expect/archive-stop-001.log

@@ -1,5 +1,5 @@
-run 001 - rmt 0, cmp 0, error version
-=====================================
+run 001 - rmt 0, cmp 0, error version, s3 0
+===========================================
 
 stanza-create db - create required data for stanza (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create

test/expect/archive-stop-002.log

@@ -1,5 +1,5 @@
-run 002 - rmt 0, cmp 1, error version
-=====================================
+run 002 - rmt 0, cmp 1, error version, s3 0
+===========================================
 
 stanza-create db - create required data for stanza (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create

test/expect/archive-stop-003.log

@@ -1,5 +1,5 @@
-run 003 - rmt 1, cmp 0, error version
-=====================================
+run 003 - rmt 1, cmp 0, error version, s3 0
+===========================================
 
 stanza-create db - create required data for stanza (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create

test/expect/archive-stop-004.log

@@ -1,5 +1,5 @@
-run 004 - rmt 1, cmp 0, error connect
-=====================================
+run 004 - rmt 1, cmp 0, error connect, s3 0
+===========================================
 
 stanza-create db - create required data for stanza (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create

test/expect/archive-stop-005.log

@@ -1,5 +1,5 @@
-run 005 - rmt 1, cmp 1, error version
-=====================================
+run 005 - rmt 1, cmp 1, error version, s3 0
+===========================================
 
 stanza-create db - create required data for stanza (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create

test/expect/archive-stop-006.log

@@ -1,5 +1,5 @@
-run 006 - rmt 1, cmp 1, error connect
-=====================================
+run 006 - rmt 1, cmp 1, error connect, s3 0
+===========================================
 
 stanza-create db - create required data for stanza (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create

test/expect/archive-stop-007.log

@@ -0,0 +1,58 @@
+run 007 - rmt 1, cmp 0, error connect, s3 1
+===========================================
+
+stanza-create db - create required data for stanza (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db --backup-host=bogus archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
+------------------------------------------------------------------------------------------------------------------------------------
+P00  ERROR: [042]: process 'bogus remote' terminated unexpectedly: ssh: Could not resolve hostname bogus: Name or service not known
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db --backup-host=bogus archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
+------------------------------------------------------------------------------------------------------------------------------------
+P00  ERROR: [042]: process 'bogus remote' terminated unexpectedly: ssh: Could not resolve hostname bogus: Name or service not known
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000004
+------------------------------------------------------------------------------------------------------------------------------------
+P00   WARN: dropped WAL file 000000010000000100000004 because archive queue exceeded 33554432 bytes
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005
+------------------------------------------------------------------------------------------------------------------------------------

test/expect/full-real-001.log

@@ -1,5 +1,5 @@
-run 001 - bkp 0, sby 0, dst db-master, asy 0, cmp 0
-===================================================
+run 001 - bkp 0, sby 0, dst db-master, asy 0, cmp 0, s3 0
+=========================================================
 
 stanza-create db - main create stanza info files (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-002.log

@@ -1,5 +1,5 @@
-run 002 - bkp 0, sby 0, dst db-master, asy 0, cmp 1
-===================================================
+run 002 - bkp 0, sby 0, dst db-master, asy 0, cmp 1, s3 0
+=========================================================
 
 stanza-create db - main create stanza info files (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-003.log

@@ -1,5 +1,5 @@
-run 003 - bkp 0, sby 0, dst db-master, asy 1, cmp 0
-===================================================
+run 003 - bkp 0, sby 0, dst db-master, asy 1, cmp 0, s3 0
+=========================================================
 
 stanza-create db - main create stanza info files (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-004.log

@@ -1,5 +1,5 @@
-run 004 - bkp 0, sby 0, dst db-master, asy 1, cmp 1
-===================================================
+run 004 - bkp 0, sby 0, dst db-master, asy 1, cmp 1, s3 0
+=========================================================
 
 stanza-create db - main create stanza info files (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-005.log

@@ -1,5 +1,5 @@
-run 005 - bkp 0, sby 1, dst db-master, asy 0, cmp 0
-===================================================
+run 005 - bkp 0, sby 1, dst db-master, asy 0, cmp 0, s3 0
+=========================================================
 
 stanza-create db - main create stanza info files (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-006.log

@@ -1,5 +1,5 @@
-run 006 - bkp 0, sby 1, dst db-standby, asy 0, cmp 0
-====================================================
+run 006 - bkp 0, sby 1, dst db-standby, asy 0, cmp 0, s3 0
+==========================================================
 
 stanza-create db - main create stanza info files (db-standby host)
 > [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-007.log

@@ -1,5 +1,5 @@
-run 007 - bkp 1, sby 0, dst backup, asy 0, cmp 0
-================================================
+run 007 - bkp 1, sby 0, dst backup, asy 0, cmp 0, s3 0
+======================================================
 
 stanza-create db - main create stanza info files (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-008.log

@@ -1,5 +1,5 @@
-run 008 - bkp 1, sby 0, dst backup, asy 0, cmp 1
-================================================
+run 008 - bkp 1, sby 0, dst backup, asy 0, cmp 1, s3 0
+======================================================
 
 stanza-create db - main create stanza info files (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-009.log

@@ -1,5 +1,5 @@
-run 009 - bkp 1, sby 0, dst backup, asy 1, cmp 0
-================================================
+run 009 - bkp 1, sby 0, dst backup, asy 1, cmp 0, s3 0
+======================================================
 
 stanza-create db - main create stanza info files (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-010.log

@@ -1,5 +1,5 @@
-run 010 - bkp 1, sby 0, dst backup, asy 1, cmp 1
-================================================
+run 010 - bkp 1, sby 0, dst backup, asy 1, cmp 1, s3 0
+======================================================
 
 stanza-create db - main create stanza info files (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-011.log

@@ -1,5 +1,5 @@
-run 011 - bkp 1, sby 1, dst backup, asy 0, cmp 0
-================================================
+run 011 - bkp 1, sby 1, dst backup, asy 0, cmp 0, s3 0
+======================================================
 
 stanza-create db - main create stanza info files (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create

test/expect/full-real-012.log

@@ -0,0 +1,387 @@
+run 012 - bkp 1, sby 0, dst backup, asy 1, cmp 1, s3 1
+======================================================
+
+stanza-create db - main create stanza info files (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on missing archive.info file (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+stanza-create db - force create stanza info files (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail  --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+
+full backup - fail on archive_mode=off (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --type=full --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on archive_mode=off (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on archive_mode=off (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+full backup - fail on invalid archive_command (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --type=full --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on invalid archive_command (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on invalid archive_command (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on archive timeout when archive-check=n (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --no-archive-check --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - verify success (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - verify success (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on archive mismatch after upgrade (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on archive mismatch after upgrade (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on archive timeout (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on archive timeout (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=0.1 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on backup info mismatch (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - fail on backup info mismatch (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - verify success after backup (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+check db - verify success after backup (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check
+------------------------------------------------------------------------------------------------------------------------------------
+
+stanza-create db - verify success with force (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail  --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+
+stanza-create db - fail on database mismatch with directory (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail  --db-path=[TEST_PATH]/db-master/db/testbase/ stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+
+stanza-create db - successfully create stanza files to be upgraded (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail  --db-path=[TEST_PATH]/db-master/db/testbase/ --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+
+stanza-upgrade db - upgrade stanza files online (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail stanza-upgrade
+------------------------------------------------------------------------------------------------------------------------------------
+
+full backup - fail on backup lock exists (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --type=full --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+
+full backup - update during backup (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf  --buffer-size=16384 --type=full --stanza=db backup --test --test-delay=1 --test-point=manifest-build=y
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
+----------------------------------------------------------
+[db]
+db-path=[TEST_PATH]/db-master/db/base
+db-port=[PORT-1]
+db-socket-path=[TEST_PATH]/db-master/db
+
+[global]
+backup-cmd=[BACKREST-BIN]
+backup-config=[TEST_PATH]/backup/pgbackrest.conf
+backup-host=backup
+backup-user=[USER-1]
+db-timeout=45
+lock-path=[TEST_PATH]/db-master/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/db-master/log
+protocol-timeout=60
+spool-path=[TEST_PATH]/db-master/spool
+
+[global:archive-push]
+archive-async=y
+
++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf
+-------------------------------------------------------
+[db]
+db-cmd=[BACKREST-BIN]
+db-config=[TEST_PATH]/db-master/pgbackrest.conf
+db-host=db-master
+db-path=[TEST_PATH]/db-master/db/base
+db-port=[PORT-1]
+db-user=[USER-2]
+
+[global]
+db-timeout=45
+lock-path=[TEST_PATH]/backup/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/backup/log
+protocol-timeout=60
+repo-path=/
+repo-s3-bucket=pgbackrest-dev
+repo-s3-endpoint=s3.amazonaws.com
+repo-s3-key=accessKey1
+repo-s3-key-secret=verySecretKey1
+repo-s3-region=us-east-1
+repo-s3-verify-ssl=n
+repo-type=s3
+
+[global:backup]
+archive-copy=y
+start-fast=y
+
+stop all stanzas (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf stop
+------------------------------------------------------------------------------------------------------------------------------------
+
+incr backup - attempt backup when stopped (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+
+start all stanzas (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf start
+------------------------------------------------------------------------------------------------------------------------------------
+
+incr backup - fail on archive_mode=always (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+
+incr backup - fail on backup already running (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+
+incr backup - update during backup (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stop-auto --no-archive-check --buffer-size=32768 --stanza=db backup --test --test-delay=1 --test-point=manifest-build=y
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
+----------------------------------------------------------
+[db]
+db-path=[TEST_PATH]/db-master/db/base
+db-port=[PORT-1]
+db-socket-path=[TEST_PATH]/db-master/db
+
+[global]
+backup-cmd=[BACKREST-BIN]
+backup-config=[TEST_PATH]/backup/pgbackrest.conf
+backup-host=backup
+backup-user=[USER-1]
+db-timeout=45
+lock-path=[TEST_PATH]/db-master/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/db-master/log
+protocol-timeout=60
+spool-path=[TEST_PATH]/db-master/spool
+
+[global:archive-push]
+archive-async=y
+
++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf
+-------------------------------------------------------
+[db]
+db-cmd=[BACKREST-BIN]
+db-config=[TEST_PATH]/db-master/pgbackrest.conf
+db-host=db-master
+db-path=[TEST_PATH]/db-master/db/base
+db-port=[PORT-1]
+db-user=[USER-2]
+
+[global]
+db-timeout=45
+lock-path=[TEST_PATH]/backup/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/backup/log
+protocol-timeout=60
+repo-path=/
+repo-s3-bucket=pgbackrest-dev
+repo-s3-endpoint=s3.amazonaws.com
+repo-s3-key=accessKey1
+repo-s3-key-secret=verySecretKey1
+repo-s3-region=us-east-1
+repo-s3-verify-ssl=n
+repo-type=s3
+
+[global:backup]
+archive-copy=y
+start-fast=y
+
+restore, type 'default', expect exit 38 - postmaster running (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --link-all --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
+restore, type 'default', expect exit 40 - path not empty (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --link-all --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
+restore, type 'default' (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf  --db-include=test1  --link-all --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
+----------------------------------------------------------------
+restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+
+restore, force, backup '[BACKUP-FULL-1]', type 'immediate' (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --force --set=[BACKUP-FULL-1] --type=immediate --link-all --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
+----------------------------------------------------------------
+restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+recovery_target = 'immediate'
+
+restore, force, backup '[BACKUP-INCR-1]', type 'xid', target '[XID-TARGET-1]', target-action=promote (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --force --set=[BACKUP-INCR-1] --tablespace-map-all=../../tablespace  --type=xid --target="[XID-TARGET-1]" --link-all --target-action=promote --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
+----------------------------------------------------------------
+restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+recovery_target_xid = '[XID-TARGET-1]'
+recovery_target_action = 'promote'
+
+restore, type 'preserve' (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --type=preserve --link-all --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
+----------------------------------------------------------------
+restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+recovery_target_xid = '[XID-TARGET-1]'
+recovery_target_action = 'promote'
+
+restore delta, backup '[BACKUP-FULL-1]', type 'time', target '[TIMESTAMP-TARGET-1]' (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-FULL-1] --type=time --target="[TIMESTAMP-TARGET-1]" --link-all --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
+----------------------------------------------------------------
+restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+recovery_target_time = '[TIMESTAMP-TARGET-1]'
+
+restore delta, backup '[BACKUP-INCR-1]', type 'xid', target '[XID-TARGET-1]', exclusive (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-INCR-1] --type=xid --target="[XID-TARGET-1]" --target-exclusive --link-all --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
+----------------------------------------------------------------
+restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+recovery_target_xid = '[XID-TARGET-1]'
+recovery_target_inclusive = 'false'
+
+restore delta, force, type 'name', target 'backrest' (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --force --type=name --target="backrest" --link-all --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
+----------------------------------------------------------------
+restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+recovery_target_name = 'backrest'
+
+restore delta, backup '[BACKUP-INCR-1]', type 'default', timeline '4' (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-INCR-1] --target-timeline="4" --link-all --stanza=db restore
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/db/base/recovery.conf
+----------------------------------------------------------------
+standby_mode = 'on'
+restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get %f "%p"'
+recovery_target_timeline = '4'
+
+incr backup - fail on --no-online (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+
+incr backup - succeed on --no-online with --force (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --force --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+
++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
+----------------------------------------------------------
+[db]
+db-path=[TEST_PATH]/db-master/db/base
+db-port=[PORT-1]
+db-socket-path=[TEST_PATH]/db-master/db
+recovery-option=standby-mode=on
+
+[global]
+backup-cmd=[BACKREST-BIN]
+backup-config=[TEST_PATH]/backup/pgbackrest.conf
+backup-host=backup
+backup-user=[USER-1]
+db-timeout=45
+lock-path=[TEST_PATH]/db-master/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/db-master/log
+protocol-timeout=60
+spool-path=[TEST_PATH]/db-master/spool
+
+[global:archive-push]
+archive-async=y
+
++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf
+-------------------------------------------------------
+[db]
+db-cmd=[BACKREST-BIN]
+db-config=[TEST_PATH]/db-master/pgbackrest.conf
+db-host=db-master
+db-path=[TEST_PATH]/db-master/db/base
+db-port=[PORT-1]
+db-user=[USER-2]
+
+[global]
+db-timeout=45
+lock-path=[TEST_PATH]/backup/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/backup/log
+protocol-timeout=60
+repo-path=/
+repo-s3-bucket=pgbackrest-dev
+repo-s3-endpoint=s3.amazonaws.com
+repo-s3-key=accessKey1
+repo-s3-key-secret=verySecretKey1
+repo-s3-region=us-east-1
+repo-s3-verify-ssl=n
+repo-type=s3
+
+[global:backup]
+archive-copy=y
+start-fast=y

test/expect/full-synthetic-001.log

@@ -1,5 +1,5 @@
-run 001 - rmt 0, cmp 0, hardlink 0
-==================================
+run 001 - rmt 0, cmp 0, hardlink 0, s3 0
+========================================
 
 info all stanzas - no stanzas exist (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info

test/expect/full-synthetic-002.log

@@ -1,5 +1,5 @@
-run 002 - rmt 0, cmp 0, hardlink 1
-==================================
+run 002 - rmt 0, cmp 0, hardlink 1, s3 0
+========================================
 
 info all stanzas - no stanzas exist (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info

test/expect/full-synthetic-003.log

@@ -1,5 +1,5 @@
-run 003 - rmt 0, cmp 1, hardlink 0
-==================================
+run 003 - rmt 0, cmp 1, hardlink 0, s3 0
+========================================
 
 info all stanzas - no stanzas exist (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info

test/expect/full-synthetic-004.log

@@ -1,5 +1,5 @@
-run 004 - rmt 0, cmp 1, hardlink 1
-==================================
+run 004 - rmt 0, cmp 1, hardlink 1, s3 0
+========================================
 
 info all stanzas - no stanzas exist (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info

test/expect/full-synthetic-005.log

@@ -1,5 +1,5 @@
-run 005 - rmt 1, cmp 0, hardlink 0
-==================================
+run 005 - rmt 1, cmp 0, hardlink 0, s3 0
+========================================
 
 info all stanzas - no stanzas exist (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info

test/expect/full-synthetic-006.log

@@ -1,5 +1,5 @@
-run 006 - rmt 1, cmp 0, hardlink 1
-==================================
+run 006 - rmt 1, cmp 0, hardlink 1, s3 0
+========================================
 
 info all stanzas - no stanzas exist (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info

test/expect/full-synthetic-007.log

@@ -1,5 +1,5 @@
-run 007 - rmt 1, cmp 1, hardlink 0
-==================================
+run 007 - rmt 1, cmp 1, hardlink 0, s3 0
+========================================
 
 info all stanzas - no stanzas exist (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info

test/expect/full-synthetic-008.log

@@ -1,5 +1,5 @@
-run 008 - rmt 1, cmp 1, hardlink 1
-==================================
+run 008 - rmt 1, cmp 1, hardlink 1, s3 0
+========================================
 
 info all stanzas - no stanzas exist (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn info

test/expect/help-help-001.log

@@ -78,6 +78,13 @@ compress=n [default=3]
   --protocol-timeout        protocol timeout [default=1830]
   --repo-path               repository path where WAL segments and backups
 stored [default=/var/lib/pgbackrest]
+  --repo-s3-bucket          s3 repository bucket
+  --repo-s3-endpoint        s3 repository endpoint
+  --repo-s3-host            s3 repository host
+  --repo-s3-key             s3 repository access key
+  --repo-s3-key-secret      s3 repository secret access key
+  --repo-s3-region          s3 repository region
+  --repo-s3-verify-ssl      verify S3 server certificate [default=y]
   --repo-type               type of storage used for the repository
 [default=posix]
   --stanza                  defines a stanza [current=main]

test/expect/stanza-create-001.log

@@ -1,5 +1,5 @@
-run 001 - remote 0
-==================
+run 001 - remote 0, s3 0
+========================
 
 stanza-create db - fail on missing control file (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create

test/expect/stanza-create-002.log

@@ -1,5 +1,5 @@
-run 002 - remote 1
-==================
+run 002 - remote 1, s3 0
+========================
 
 stanza-create db - fail on missing control file (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create

test/expect/stanza-create-003.log

@@ -0,0 +1,456 @@
+run 003 - remote 0, s3 1
+========================
+
+stanza-create db - fail on missing control file (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00  ERROR: [041]: unable to open [TEST_PATH]/db-master/db/base/global/pg_control
+P00   INFO: stanza-create command end: aborted with exception [041]
+
+stanza-create db - successfully create the stanza (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - successful rerun of stanza-create (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: archive-push command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00  DEBUG:     Archive::ArchivePush->process(): strWalPathFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+P00  DEBUG:     Archive::ArchivePushFile::archivePushFile(): bCompress = true, strWalFile = 000000010000000100000001, strWalPath = [TEST_PATH]/db-master/db/base/pg_xlog
+P00  DEBUG:     Storage::Local->new(): bAllowTemp = <true>, hRule = [hash], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp
+P00  DEBUG:     Archive::ArchiveCommon::walInfo(): strWalFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+P00  DEBUG:     Archive::ArchiveCommon::walInfo=>: strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00  DEBUG:     Archive::ArchivePushFile::archivePushCheck(): strArchiveFile = 000000010000000100000001, strDbVersion = 9.4, strWalFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, ullDbSysId = 6353949018581704918
+P00  DEBUG:     Archive::ArchiveInfo->new(): bIgnoreMissing = <false>, bLoad = <true>, bRequired = <true>, strArchiveClusterPath = /archive/db
+P00  DEBUG:     Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], xFileExp = /archive/db/archive.info
+P00  DEBUG:     Archive::ArchiveInfo->check(): bRequired = <true>, strDbVersion = 9.4, ullDbSysId = 6353949018581704918
+P00  DEBUG:     Archive::ArchiveInfo->archiveId(): strDbVersion = [undef], ullDbSysId = [undef]
+P00  DEBUG:     Archive::ArchiveInfo->archiveId=>: strArchiveId = 9.4-1
+P00  DEBUG:     Archive::ArchiveInfo->check=>: strArchiveId = 9.4-1
+P00  DEBUG:     Archive::ArchiveCommon::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000001
+P00  DEBUG:     Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPathExp = <REPO:ARCHIVE>/9.4-1/0000000100000001, strSortOrder = <forward>
+P00  DEBUG:     Storage::Local->list=>: stryFileList = ()
+P00  DEBUG:     Archive::ArchiveCommon::walSegmentFind=>: strWalFileName = [undef]
+P00  DEBUG:     Archive::ArchivePushFile::archivePushCheck=>: strArchiveId = 9.4-1, strChecksum = [undef], strWarning = [undef]
+P00  DEBUG:     Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
+P00  DEBUG:     Storage::Local->hashSize(): xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+P00  DEBUG:     Storage::Local->openRead(): bIgnoreMissing = <false>, rhyFilter = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+P00  DEBUG:     Storage::Local->hashSize=>: lSize = 16777216, strHash = 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27
+P00  DEBUG:     Storage::Local->openRead(): bIgnoreMissing = <false>, rhyFilter = ({strClass => pgBackRest::Storage::Filter::Gzip}), xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+P00  DEBUG:     Storage::Local->openWrite(): bAtomic = true, bPathCreate = true, lTimestamp = [undef], rhyFilter = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = <REPO:ARCHIVE>/9.4-1/000000010000000100000001-1e34fa1c833090d94b9bb14f2a8d3153dca6ea27.gz
+P00  DEBUG:     Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
+P00  DEBUG:     Archive::ArchivePushFile::archivePushFile=>: strWarning = [undef]
+P00   INFO: pushed WAL segment 000000010000000100000001
+P00  DEBUG:     Common::Exit::exitSafe(): iExitCode = 0, oException = [undef], strSignal = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
+P00  DEBUG:     Protocol::Helper::protocolDestroy=>: iExitStatus = 0
+P00  DEBUG:     Common::Lock::lockRelease(): bFailOnNoLock = false
+P00   INFO: archive-push command end: completed successfully
+P00  DEBUG:     Common::Exit::exitSafe=>: iExitCode = 0
+
+stanza-create db - fail on archive info file missing from non-empty dir (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00  ERROR: [055]: archive information missing
+            HINT: use stanza-create --force to force the stanza data to be created.
+P00   INFO: stanza-create command end: aborted with exception [055]
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - force create archive.info from gz file (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - repeat create (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - hash check fails requiring force (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   WARN: unable to create stanza 'db'
+P00  ERROR: [028]: archive info file invalid
+            HINT: use stanza-upgrade if the database has been upgraded or use --force
+P00   INFO: stanza-create command end: aborted with exception [028]
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="8.0"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - use force to overwrite the invalid file (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - fail on database mismatch without force option (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   WARN: unable to create stanza 'db'
+P00  ERROR: [028]: archive info file invalid
+            HINT: use stanza-upgrade if the database has been upgraded or use --force
+P00   INFO: stanza-create command end: aborted with exception [028]
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - force create archive.info from uncompressed file (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - force with missing WAL archive file (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - force with missing WAL archive directory (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6353949018581704918,"db-version":"9.4"}

test/expect/stanza-upgrade-001.log

@@ -1,5 +1,5 @@
-run 001 - local
-===============
+run 001 - remote 0, s3 0
+========================
 
 stanza-upgrade db - fail on stanza not initialized since archive.info is missing (db-master host)
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade

test/expect/stanza-upgrade-002.log

@@ -1,5 +1,5 @@
-run 002 - remote
-================
+run 002 - remote 1, s3 0
+========================
 
 stanza-upgrade db - fail on stanza not initialized since archive.info is missing (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade

test/expect/stanza-upgrade-003.log

@@ -0,0 +1,537 @@
+run 003 - remote 1, s3 1
+========================
+
+stanza-upgrade db - fail on stanza not initialized since archive.info is missing (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00  ERROR: [055]: archive.info does not exist but is required to push/get WAL segments
+            HINT: is archive_command configured in postgresql.conf?
+            HINT: has a stanza-create been performed?
+            HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
+P00   INFO: stanza-upgrade command end: aborted with exception [055]
+
+stanza-create db - successfully create the stanza (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201306121
+db-control-version=937
+db-id=1
+db-system-id=6395542721432104958
+db-version="9.3"
+
+[db:history]
+1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6395542721432104958
+db-version="9.3"
+
+[db:history]
+1={"db-id":6395542721432104958,"db-version":"9.3"}
+
+stanza-upgrade db - already up to date (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: the stanza data is already up to date
+P00   INFO: stanza-upgrade command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201306121
+db-control-version=937
+db-id=1
+db-system-id=6395542721432104958
+db-version="9.3"
+
+[db:history]
+1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6395542721432104958
+db-version="9.3"
+
+[db:history]
+1={"db-id":6395542721432104958,"db-version":"9.3"}
+
+stanza-upgrade db - fail on stanza not initialized since backup.info is missing (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00  ERROR: [055]: /backup/db/backup.info does not exist and is required to perform a backup.
+            HINT: has a stanza-create been performed?
+P00   INFO: stanza-upgrade command end: aborted with exception [055]
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6395542721432104958
+db-version="9.3"
+
+[db:history]
+1={"db-id":6395542721432104958,"db-version":"9.3"}
+
+stanza-create db - use force to recreate the stanza (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201306121
+db-control-version=937
+db-id=1
+db-system-id=6395542721432104958
+db-version="9.3"
+
+[db:history]
+1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=1
+db-system-id=6395542721432104958
+db-version="9.3"
+
+[db:history]
+1={"db-id":6395542721432104958,"db-version":"9.3"}
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+P00  ERROR: [044]: raised on 'backup remote' host: WAL segment version 9.4 does not match archive version 9.3
+            WAL segment system-id 6353949018581704918 does not match archive system-id 6395542721432104958
+            HINT: are you archiving to the correct stanza?
+
+stanza-upgrade db - successful upgrade creates mismatched files (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-upgrade command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=2
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"}
+2={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=2
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-id":6395542721432104958,"db-version":"9.3"}
+2={"db-id":6353949018581704918,"db-version":"9.4"}
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+
+stanza-create db - use force to recreate the stanza producing mismatched info history but same current db-id (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=2
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201306121,"db-control-version":937,"db-system-id":6395542721432104958,"db-version":"9.3"}
+2={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=2
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+2={"db-id":6353949018581704918,"db-version":"9.4"}
+
+stanza-create db - use force to recreate the stanza producing mismatched db-id (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-create command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-catalog-version=201409291
+db-control-version=942
+db-id=1
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=2
+db-system-id=6353949018581704918
+db-version="9.4"
+
+[db:history]
+2={"db-id":6353949018581704918,"db-version":"9.4"}
+
+full backup - create first full backup  (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --retention-full=2 --no-online --log-level-console=detail --type=full --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: backup command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --retention-full=2 --stanza=db --start-fast --type=full
+P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 1e34fa1c833090d94b9bb14f2a8d3153dca6ea27
+P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/global/pg_control (8KB, 100%) checksum 89373d9f2973502940de06bc5212489df3f8a912
+P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
+P00   INFO: full backup size = 16MB
+P00   INFO: new backup label = [BACKUP-FULL-1]
+P00   INFO: backup command end: completed successfully
+P00   INFO: expire command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --retention-archive=2 --retention-full=2 --stanza=db
+P00   INFO: full backup total < 2 - using oldest full backup for 9.4-2 archive retention
+P00   INFO: expire command end: completed successfully
+
++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
+----------------------------------------------------------
+[db]
+db-path=[TEST_PATH]/db-master/db/base
+
+[global]
+backup-cmd=[BACKREST-BIN]
+backup-config=[TEST_PATH]/backup/pgbackrest.conf
+backup-host=backup
+backup-user=[USER-2]
+db-timeout=45
+lock-path=[TEST_PATH]/db-master/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/db-master/log
+protocol-timeout=60
+
++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf
+-------------------------------------------------------
+[db]
+db-cmd=[BACKREST-BIN]
+db-config=[TEST_PATH]/db-master/pgbackrest.conf
+db-host=db-master
+db-path=[TEST_PATH]/db-master/db/base
+db-user=[USER-1]
+
+[global]
+db-timeout=45
+lock-path=[TEST_PATH]/backup/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/backup/log
+protocol-timeout=60
+repo-path=/
+repo-s3-bucket=pgbackrest-dev
+repo-s3-endpoint=s3.amazonaws.com
+repo-s3-key=accessKey1
+repo-s3-key-secret=verySecretKey1
+repo-s3-region=us-east-1
+repo-s3-verify-ssl=n
+repo-type=s3
+
+[global:backup]
+archive-copy=y
+start-fast=y
+
+stanza-upgrade db - successfully upgrade with XX.Y-Z (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-upgrade
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: stanza-upgrade command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --stanza=db
+P00   INFO: stanza-upgrade command end: completed successfully
+
++ supplemental file: /backup/db/backup.info
+-------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[backup:current]
+[BACKUP-FULL-1]={"backrest-format":5,"backrest-version":"[VERSION-1]","backup-archive-start":null,"backup-archive-stop":null,"backup-info-repo-size":[SIZE],"backup-info-repo-size-delta":[DELTA],"backup-info-size":[SIZE],"backup-info-size-delta":[DELTA],"backup-timestamp-start":[TIMESTAMP],"backup-timestamp-stop":[TIMESTAMP],"backup-type":"full","db-id":1,"option-archive-check":true,"option-archive-copy":true,"option-backup-standby":false,"option-checksum-page":false,"option-compress":true,"option-hardlink":false,"option-online":false}
+
+[db]
+db-catalog-version=201510051
+db-control-version=942
+db-id=2
+db-system-id=6392579261579036436
+db-version="9.5"
+
+[db:history]
+1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"}
+2={"db-catalog-version":201510051,"db-control-version":942,"db-system-id":6392579261579036436,"db-version":"9.5"}
+
++ supplemental file: /archive/db/archive.info
+---------------------------------------------
+[backrest]
+backrest-checksum="[CHECKSUM]"
+backrest-format=5
+backrest-version="[VERSION-1]"
+
+[db]
+db-id=4
+db-system-id=6392579261579036436
+db-version="9.5"
+
+[db:history]
+2={"db-id":6353949018581704918,"db-version":"9.4"}
+3={"db-id":35184388866048,"db-version":"10.0"}
+4={"db-id":6392579261579036436,"db-version":"9.5"}
+
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+------------------------------------------------------------------------------------------------------------------------------------
+
+diff backup - diff changed to full backup (backup host)
+> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --retention-full=2 --no-online --log-level-console=detail --type=diff --stanza=db backup
+------------------------------------------------------------------------------------------------------------------------------------
+P00   INFO: backup command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-timeout=45 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --no-online --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --retention-full=2 --stanza=db --start-fast --type=diff
+P00   WARN: no prior backup exists, diff backup has been changed to full
+P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 15b1a1a35c26b17570aca7920980f0ad11c6d858
+P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/global/pg_control (8KB, 100%) checksum e28bf39d0a56bf9fabd4049b329fcae8878bfec6
+P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
+P00   INFO: full backup size = 16MB
+P00   INFO: new backup label = [BACKUP-FULL-2]
+P00   INFO: backup command end: completed successfully
+P00   INFO: expire command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --protocol-timeout=60 --repo-path=/ --repo-s3-bucket=pgbackrest-dev --repo-s3-endpoint=s3.amazonaws.com --repo-s3-region=us-east-1 --no-repo-s3-verify-ssl --repo-type=s3 --retention-archive=2 --retention-full=2 --stanza=db
+P00   INFO: remove archive path: /archive/db/10.0-3
+P00   INFO: expire command end: completed successfully
+
++ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
+----------------------------------------------------------
+[db]
+db-path=[TEST_PATH]/db-master/db/base
+
+[global]
+backup-cmd=[BACKREST-BIN]
+backup-config=[TEST_PATH]/backup/pgbackrest.conf
+backup-host=backup
+backup-user=[USER-2]
+db-timeout=45
+lock-path=[TEST_PATH]/db-master/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/db-master/log
+protocol-timeout=60
+
++ supplemental file: [TEST_PATH]/backup/pgbackrest.conf
+-------------------------------------------------------
+[db]
+db-cmd=[BACKREST-BIN]
+db-config=[TEST_PATH]/db-master/pgbackrest.conf
+db-host=db-master
+db-path=[TEST_PATH]/db-master/db/base
+db-user=[USER-1]
+
+[global]
+db-timeout=45
+lock-path=[TEST_PATH]/backup/lock
+log-level-console=debug
+log-level-file=trace
+log-level-stderr=off
+log-path=[TEST_PATH]/backup/log
+protocol-timeout=60
+repo-path=/
+repo-s3-bucket=pgbackrest-dev
+repo-s3-endpoint=s3.amazonaws.com
+repo-s3-key=accessKey1
+repo-s3-key-secret=verySecretKey1
+repo-s3-region=us-east-1
+repo-s3-verify-ssl=n
+repo-type=s3
+
+[global:backup]
+archive-copy=y
+start-fast=y
+
+info all stanzas - db upgraded - db-1 and db-2 listed (db-master host)
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --output=json info
+------------------------------------------------------------------------------------------------------------------------------------
+[
+    {
+        "archive" : {
+            "max" : "000000010000000100000001",
+            "min" : "000000010000000100000001"
+        },
+        "backup" : [
+            {
+                "archive" : {
+                    "start" : null,
+                    "stop" : null
+                },
+                "backrest" : {
+                    "format" : 5,
+                    "version" : "[VERSION-1]"
+                },
+                "database" : {
+                    "id" : 1
+                },
+                "info" : {
+                    "delta" : [DELTA],
+                    "repository" : {
+                        "delta" : [DELTA],
+                        "size" : [SIZE]
+                    },
+                    "size" : [SIZE]
+                },
+                "label" : "[BACKUP-FULL-1]",
+                "prior" : null,
+                "reference" : null,
+                "timestamp" : {
+                    "start" : [TIMESTAMP],
+                    "stop" : [TIMESTAMP]
+                },
+                "type" : "full"
+            },
+            {
+                "archive" : {
+                    "start" : null,
+                    "stop" : null
+                },
+                "backrest" : {
+                    "format" : 5,
+                    "version" : "[VERSION-1]"
+                },
+                "database" : {
+                    "id" : 2
+                },
+                "info" : {
+                    "delta" : [DELTA],
+                    "repository" : {
+                        "delta" : [DELTA],
+                        "size" : [SIZE]
+                    },
+                    "size" : [SIZE]
+                },
+                "label" : "[BACKUP-FULL-2]",
+                "prior" : null,
+                "reference" : null,
+                "timestamp" : {
+                    "start" : [TIMESTAMP],
+                    "stop" : [TIMESTAMP]
+                },
+                "type" : "full"
+            }
+        ],
+        "db" : [
+            {
+                "id" : "1",
+                "system-id" : 6353949018581704918,
+                "version" : "9.4"
+            },
+            {
+                "id" : "2",
+                "system-id" : 6392579261579036436,
+                "version" : "9.5"
+            }
+        ],
+        "name" : "db",
+        "status" : {
+            "code" : 0,
+            "message" : "ok"
+        }
+    }
+]

test/lib/pgBackRestTest/Common/CiTest.pm

@@ -163,6 +163,14 @@ sub process
             " libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man" .
             " devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl python-pip\n" .
         "  - |\n" .
+        "    # Install & Configure AWS CLI\n" .
+        "    pip install --upgrade --user awscli\n" .
+        "    aws configure set region us-east-1\n" .
+        "    aws configure set aws_access_key_id accessKey1\n" .
+        "    aws configure set aws_secret_access_key verySecretKey1\n" .
+        "    aws help --version\n" .
+        "    aws configure list\n" .
+        "  - |\n" .
         "    # Build Devel::Cover\n" .
         "    git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl\n" .
         '    cd ~/libdevel-cover-perl && git checkout debian/' . LIB_COVER_VERSION . " && debuild -i -us -uc -b\n" .

test/lib/pgBackRestTest/Common/ContainerTest.pm

@@ -288,6 +288,109 @@ sub coverSetup
     return $strScript;
 }
 
+####################################################################################################################################
+# S3 server setup
+####################################################################################################################################
+sub s3ServerSetup
+{
+    my $strOS = shift;
+
+    # Install node.js
+    my $strScript =
+        "\n\n# Install node.js\n";
+
+    if ($strOS eq VM_U16)
+    {
+        $strScript .=
+            "RUN wget -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x && \\\n" .
+            "    bash /root/nodejs.sh && \\\n" .
+            "    apt-get install -y nodejs";
+    }
+    elsif ($strOS eq VM_CO7)
+    {
+        $strScript .=
+            "RUN wget -O /root/nodejs.sh https://rpm.nodesource.com/setup_6.x && \\\n" .
+            "    bash /root/nodejs.sh && \\\n" .
+            "    yum install -y nodejs";
+    }
+
+    # Install Scality S3
+    $strScript .=
+        "\n\n# Install Scality S3\n";
+
+    if ($strOS eq VM_U16)
+    {
+        $strScript .=
+            "RUN apt-get install -y python build-essential git && \\\n";
+    }
+    elsif ($strOS eq VM_CO7)
+    {
+        $strScript .=
+            "RUN yum install -y python git && \\\n";
+    }
+
+    $strScript .=
+        "    wget -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz && \\\n" .
+        "    mkdir /root/scalitys3 && \\\n" .
+        "    tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz && \\\n" .
+        "    cd /root/scalitys3 && \\\n" .
+        "    npm install && \\\n" .
+        "    openssl genrsa -out ca.key 2048 && \\\n" .
+        "    openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999" .
+            " -subj \"/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org\" && \\\n" .
+        "    openssl genrsa -out server.key 2048 && \\\n" .
+        "    openssl req -new -key server.key -out server.csr" .
+            " -subj \"/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org\" && \\\n" .
+        "    openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999" .
+            " -sha256 && \\\n" .
+        '    sed -i "0,/,/s//,\n    \"certFilePaths\": { \"key\": \".\/server.key\", \"cert\":' .
+            ' \".\/server.crt\", \"ca\": \".\/ca.crt\" },/" ./config.json' . " && \\\n" .
+        '    sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json';
+
+    return $strScript;
+}
+
+
+####################################################################################################################################
+# S3 CLI setup
+####################################################################################################################################
+sub s3CliSetup
+{
+    my $strOS = shift;
+
+    my $strScript = '';
+
+    if ($strOS eq VM_U14 || $strOS eq VM_U16 || $strOS eq VM_CO6 || $strOS eq VM_CO7)
+    {
+        # Install AWS CLI
+        my $strAwsPath = '/home/' . TEST_USER . '/.aws';
+
+        $strScript =
+            "\n\n# Install AWS CLI\n";
+
+        if ($strOS eq VM_U14 || $strOS eq VM_U16)
+        {
+            $strScript .=
+                "RUN apt-get install -y python-pip && \\\n";
+        }
+        elsif ($strOS eq VM_CO6 || $strOS eq VM_CO7)
+        {
+            $strScript .=
+                "RUN yum -y install epel-release && \\\n" .
+                "    yum -y update && \\\n" .
+                "    yum -y install python-pip && \\\n";
+        }
+
+        $strScript .=
+            "    pip install --upgrade awscli && \\\n" .
+            '    sudo -i -u ' . TEST_USER . " aws configure set region us-east-1 && \\\n" .
+            '    sudo -i -u ' . TEST_USER . " aws configure set aws_access_key_id accessKey1 && \\\n" .
+            '    sudo -i -u ' . TEST_USER . " aws configure set aws_secret_access_key verySecretKey1";
+    }
+
+    return $strScript;
+}
+
 ####################################################################################################################################
 # Install Perl packages
 ####################################################################################################################################
@@ -312,7 +415,7 @@ sub perlInstall
             $strImage .= ' perl-JSON-PP';
         }
 
-        $strImage .= ' perl-Digest-SHA perl-DBD-Pg';
+        $strImage .= ' perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL';
     }
     elsif ($oVm->{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN)
     {
@@ -323,6 +426,8 @@ sub perlInstall
         {
             $strImage .= ' libnet-daemon-perl libplrpc-perl';
         }
+
+        $strImage .= ' libio-socket-ssl-perl libxml-libxml-perl libhtml-parser-perl';
     }
     else
     {
@@ -396,7 +501,7 @@ sub containerBuild
 
         if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL)
         {
-            $strScript .= "RUN yum -y install openssh-server openssh-clients\n";
+            $strScript .= "RUN yum -y install openssh-server openssh-clients wget\n";
         }
         elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN)
         {
@@ -563,6 +668,32 @@ sub containerBuild
             executeTest('docker rm -f test-build');
         }
 
+        # S3 image
+        ###########################################################################################################################
+        $strImageParent = $oVm->{&VM_U16}{&VM_IMAGE};
+        $strImage = "${strOS}-s3-server";
+
+        # Required packages
+        $strScript =
+            "RUN apt-get update && \\\n" .
+            "    apt-get install -y wget";
+
+        # Setup S3 server
+        $strScript .= s3ServerSetup(VM_U16);
+
+        # Fix root tty
+        $strScript .=
+            "\n\n# Fix root tty\n" .
+            "RUN echo 'tty -s && mesg n || true' > /root/.profile";
+
+        # Set entrypoint
+        $strScript .=
+            "\n\nENTRYPOINT npm start --prefix /root/scalitys3";
+
+        # Write the image
+        containerWrite(
+            $oStorageDocker, $strTempPath, $strOS, 'S3 Server', $strImageParent, $strImage, $strScript, $bVmForce, false);
+
         # Db image
         ###########################################################################################################################
         my @stryDbBuild;
@@ -640,6 +771,9 @@ sub containerBuild
                 # Setup sudo
                 $strScript .= "\n\n" . sudoSetup($strOS, TEST_GROUP);
 
+                # Setup s3 cli
+                $strScript .= s3CliSetup($strOS);
+
                 # Write the image
                 containerWrite(
                     $oStorageDocker, $strTempPath, $strOS, "${strTitle} Doc", $strImageParent, $strImage, $strScript, $bVmForce);
@@ -705,6 +839,15 @@ sub containerBuild
         # Setup Devel::Cover
         $strScript .= coverSetup($strOS);
 
+        # Setup S3 server
+        if ($strOS eq VM_U16 || $strOS eq VM_CO7)
+        {
+            $strScript .= s3ServerSetup($strOS);
+        }
+
+        # Setup S3 CLI
+        $strScript .= s3CliSetup($strOS);
+
         # Write the image
         containerWrite(
             $oStorageDocker, $strTempPath, $strOS, 'Loop Test', $strImageParent, $strImage, $strScript, $bVmForce, true, true);

test/lib/pgBackRestTest/Common/DefineTest.pm

@@ -16,6 +16,8 @@ use Exporter qw(import);
 use pgBackRest::Common::Log;
 use pgBackRest::Common::String;
 
+use pgBackRestTest::Common::VmTest;
+
 ################################################################################################################################
 # Test definition constants
 ################################################################################################################################
@@ -47,6 +49,9 @@ use constant TESTDEF_PROCESS                                        => 'process'
 # Total runs in the test
 use constant TESTDEF_TOTAL                                          => 'total';
     push @EXPORT, qw(TESTDEF_TOTAL);
+# VMs that the test can run on
+use constant TESTDEF_VM                                             => 'vm';
+    push @EXPORT, qw(TESTDEF_VM);
 
 # The test provides full coverage for the module
 use constant TESTDEF_COVERAGE_FULL                                  => true;
@@ -206,6 +211,27 @@ my $oTestDef =
                         'Storage/Posix/FileWrite' => TESTDEF_COVERAGE_PARTIAL,
                     },
                 },
+                {
+                    &TESTDEF_NAME => 's3-auth',
+                    &TESTDEF_TOTAL => 5,
+
+                    &TESTDEF_COVERAGE =>
+                    {
+                        'Storage/S3/Auth' => TESTDEF_COVERAGE_FULL,
+                    },
+                },
+                {
+                    &TESTDEF_NAME => 's3',
+                    &TESTDEF_TOTAL => 7,
+                    &TESTDEF_VM => [VM_CO7, VM_U16],
+
+                    &TESTDEF_COVERAGE =>
+                    {
+                        'Storage/S3/Driver' => TESTDEF_COVERAGE_PARTIAL,
+                        'Storage/S3/FileRead' => TESTDEF_COVERAGE_PARTIAL,
+                        'Storage/S3/FileWrite' => TESTDEF_COVERAGE_FULL,
+                    },
+                },
                 {
                     &TESTDEF_NAME => 'local',
                     &TESTDEF_TOTAL => 9,
@@ -292,20 +318,20 @@ my $oTestDef =
                 },
                 {
                     &TESTDEF_NAME => 'push',
-                    &TESTDEF_TOTAL => 8,
+                    &TESTDEF_TOTAL => 9,
                     &TESTDEF_PROCESS => true,
                     &TESTDEF_INDIVIDUAL => true,
                     &TESTDEF_EXPECT => true,
                 },
                 {
                     &TESTDEF_NAME => 'stop',
-                    &TESTDEF_TOTAL => 6,
+                    &TESTDEF_TOTAL => 7,
                     &TESTDEF_INDIVIDUAL => true,
                     &TESTDEF_EXPECT => true,
                 },
                 {
                     &TESTDEF_NAME => 'get',
-                    &TESTDEF_TOTAL => 8,
+                    &TESTDEF_TOTAL => 9,
                     &TESTDEF_INDIVIDUAL => true,
                     &TESTDEF_EXPECT => true,
                 },
@@ -370,13 +396,13 @@ my $oTestDef =
                 },
                 {
                     &TESTDEF_NAME => 'create',
-                    &TESTDEF_TOTAL => 2,
+                    &TESTDEF_TOTAL => 3,
                     &TESTDEF_EXPECT => true,
                     &TESTDEF_INDIVIDUAL => true,
                 },
                 {
                     &TESTDEF_NAME => 'upgrade',
-                    &TESTDEF_TOTAL => 2,
+                    &TESTDEF_TOTAL => 3,
                     &TESTDEF_EXPECT => true,
                     &TESTDEF_INDIVIDUAL => true,
                 },
@@ -393,11 +419,11 @@ my $oTestDef =
             [
                 {
                     &TESTDEF_NAME => 'synthetic',
-                    &TESTDEF_TOTAL => 8,
+                    &TESTDEF_TOTAL => 9,
                 },
                 {
                     &TESTDEF_NAME => 'real',
-                    &TESTDEF_TOTAL => 11,
+                    &TESTDEF_TOTAL => 12,
                     &TESTDEF_DB => true,
                 }
             ]
@@ -432,10 +458,10 @@ foreach my $hModule (@{$oTestDef->{&TESTDEF_MODULE}})
 
         # Resolve variables that can be set in the module or the test
         foreach my $strVar (
-            TESTDEF_CONTAINER, TESTDEF_EXPECT, TESTDEF_PROCESS, TESTDEF_DB, TESTDEF_INDIVIDUAL)
+            TESTDEF_CONTAINER, TESTDEF_EXPECT, TESTDEF_PROCESS, TESTDEF_DB, TESTDEF_INDIVIDUAL, TESTDEF_VM)
         {
             $hTestDefHash->{$strModule}{$strTest}{$strVar} = coalesce(
-                $hModuleTest->{$strVar}, coalesce($hModule->{$strVar}, false));
+                $hModuleTest->{$strVar}, $hModule->{$strVar}, $strVar eq TESTDEF_VM ? undef : false);
         }
 
         # Set test count

test/lib/pgBackRestTest/Common/ExecuteTest.pm

@@ -100,7 +100,7 @@ sub begin
     $self->{pId} = open3(undef, $self->{hOut}, $self->{hError}, $self->{strCommand});
 
     # Create select objects
-    $self->{oIo} = new pgBackRest::Common::Io::Buffered(new pgBackRest::Common::Io::Handle('exec test', $self->{hOut}), 120, 65536);
+    $self->{oIo} = new pgBackRest::Common::Io::Buffered(new pgBackRest::Common::Io::Handle('exec test', $self->{hOut}), 540, 65536);
 
     if (!defined($self->{hError}))
     {

test/lib/pgBackRestTest/Common/FileTest.pm

@@ -28,13 +28,17 @@ use pgBackRest::Common::Wait;
 use pgBackRest::Config::Config;
 use pgBackRest::Manifest;
 use pgBackRest::Storage::Local;
+use pgBackRest::Storage::S3::Driver;
 
 use pgBackRestTest::Common::ExecuteTest;
+use pgBackRestTest::Common::HostGroupTest;
 use pgBackRestTest::Common::LogTest;
 use pgBackRestTest::Common::VmTest;
+use pgBackRestTest::Env::Host::HostBaseTest;
 use pgBackRestTest::Env::Host::HostBackupTest;
 use pgBackRestTest::Env::Host::HostDbCommonTest;
 use pgBackRestTest::Env::Host::HostDbTest;
+use pgBackRestTest::Env::Host::HostS3Test;
 
 ####################################################################################################################################
 # testLinkCreate
@@ -160,7 +164,11 @@ sub forceStorageMode
             {name => 'bRecurse', optional => true, default => false},
         );
 
-    executeTest('sudo chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp));
+    # Mode commands are ignored on S3
+    if ($oStorage->driver()->className() ne STORAGE_S3_DRIVER)
+    {
+        executeTest('sudo chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp));
+    }
 
     # Return from function and log return values if any
     return logDebugReturn($strOperation);
@@ -189,7 +197,18 @@ sub forceStorageMove
             {name => 'strDestinationPathExp'},
         );
 
-    executeTest('sudo mv ' . $oStorage->pathGet($strSourcePathExp) . ' ' . $oStorage->pathGet($strDestinationPathExp));
+    # If S3 then use storage commands to remove
+    if ($oStorage->driver()->className() eq STORAGE_S3_DRIVER)
+    {
+        hostGroupGet()->hostGet(HOST_S3)->executeS3(
+            'mv --recursive s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strSourcePathExp) .
+                ' s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strDestinationPathExp));
+    }
+    # Else remove using filesystem commands
+    else
+    {
+        executeTest('sudo mv ' . $oStorage->pathGet($strSourcePathExp) . ' ' . $oStorage->pathGet($strDestinationPathExp));
+    }
 
     # Return from function and log return values if any
     return logDebugReturn($strOperation);
@@ -220,7 +239,11 @@ sub forceStorageOwner
             {name => 'bRecurse', optional => true, default => false},
         );
 
-    executeTest('sudo chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp));
+    # Mode commands are ignored on S3
+    if ($oStorage->driver()->className() ne STORAGE_S3_DRIVER)
+    {
+        executeTest('sudo chown ' . ($bRecurse ? '-R ' : '') . "${strOwner} " . $oStorage->pathGet($strPathExp));
+    }
 
     # Return from function and log return values if any
     return logDebugReturn($strOperation);
@@ -249,7 +272,16 @@ sub forceStorageRemove
             {name => 'bRecurse', optional => true, default => false},
         );
 
-    executeTest('sudo rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp));
+    # If S3 then use storage commands to remove
+    if ($oStorage->driver()->className() eq STORAGE_S3_DRIVER)
+    {
+        $oStorage->remove($strPathExp, {bRecurse => $bRecurse});
+    }
+    # Else remove using filesystem commands
+    else
+    {
+        executeTest('sudo rm -f' . ($bRecurse ? 'r ' : ' ') . $oStorage->pathGet($strPathExp));
+    }
 
     # Return from function and log return values if any
     return logDebugReturn($strOperation);

test/lib/pgBackRestTest/Common/ListTest.pm

@@ -102,6 +102,9 @@ sub testListGet
 
                         my $bFirstDbVersion = true;
 
+                        # Skip this test if it can't run on this VM
+                        next if (defined($hTest->{&TESTDEF_VM}) && grep(/^$strTestOS$/i, @{$hTest->{&TESTDEF_VM}}) == 0);
+
                         for (my $iDbVersionIdx = $iDbVersionMax; $iDbVersionIdx >= $iDbVersionMin; $iDbVersionIdx--)
                         {
                             if ($iDbVersionIdx == -1 || $strDbVersion eq 'all' || $strDbVersion eq 'minimal' ||

test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm

@@ -26,9 +26,11 @@ use pgBackRest::Config::Config;
 use pgBackRest::Manifest;
 use pgBackRest::Protocol::Storage::Helper;
 use pgBackRest::Storage::Posix::Driver;
+use pgBackRest::Storage::S3::Driver;
 use pgBackRest::Version;
 
 use pgBackRestTest::Env::Host::HostBaseTest;
+use pgBackRestTest::Env::Host::HostS3Test;
 use pgBackRestTest::Common::ContainerTest;
 use pgBackRestTest::Common::ExecuteTest;
 use pgBackRestTest::Common::HostGroupTest;
@@ -87,10 +89,19 @@ sub new
     my $self = $class->SUPER::new($strName, {strImage => $strImage, strUser => $strUser});
     bless $self, $class;
 
-    # Create repo path
-    $self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO;
+    # If repo is on local filesystem then set the repo-path locally
+    if ($oParam->{bRepoLocal})
+    {
+        $self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO;
+    }
+    # Else on KV store and repo will be in root
+    else
+    {
+        $self->{strRepoPath} = '/';
+    }
 
-    if ($$oParam{strBackupDestination} eq $self->nameGet())
+    # Create the repo-path if on a local filesystem
+    if ($$oParam{strBackupDestination} eq $self->nameGet() && $oParam->{bRepoLocal})
     {
         storageTest()->pathCreate($self->repoPath(), {strMode => '0770'});
     }
@@ -939,6 +950,18 @@ sub configCreate
     {
         $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_PATH} = $self->repoPath();
 
+        # S3 settings
+        if ($oParam->{bS3})
+        {
+            $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_TYPE} = REPO_TYPE_S3;
+            $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_KEY} = HOST_S3_ACCESS_KEY;
+            $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_KEY_SECRET} = HOST_S3_ACCESS_SECRET_KEY;
+            $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_BUCKET} = HOST_S3_BUCKET;
+            $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_ENDPOINT} = HOST_S3_ENDPOINT;
+            $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_REGION} = HOST_S3_REGION;
+            $oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_S3_VERIFY_SSL} = 'n';
+        }
+
         if (defined($$oParam{bHardlink}) && $$oParam{bHardlink})
         {
             $self->{bHardLink} = true;
@@ -1157,15 +1180,21 @@ sub infoMunge
     }
 
     # Modify the file/directory permissions so it can be saved
+    if ($self->isFS())
+    {
         executeTest("sudo rm -f ${strFileName}* && sudo chmod 770 " . dirname($strFileName));
+    }
 
     # Save the munged data to the file
     $oMungeIni->save();
 
     # Fix permissions
+    if ($self->isFS())
+    {
         executeTest(
             "sudo chmod 640 ${strFileName}* && sudo chmod 750 " . dirname($strFileName) .
             ' && sudo chown ' . $self->userGet() . " ${strFileName}*");
+    }
 
     # Clear the cache is requested
     if (!$bCache)
@@ -1207,18 +1236,24 @@ sub infoRestore
         if ($bSave)
         {
             # Modify the file/directory permissions so it can be saved
+            if ($self->isFS())
+            {
                 executeTest("sudo rm -f ${strFileName}* && sudo chmod 770 " . dirname($strFileName));
+            }
 
             # Save the munged data to the file
             $self->{hInfoFile}{$strFileName}->{bModified} = true;
             $self->{hInfoFile}{$strFileName}->save();
 
             # Fix permissions
+            if ($self->isFS())
+            {
                 executeTest(
                     "sudo chmod 640 ${strFileName} && sudo chmod 750 " . dirname($strFileName) .
                     ' && sudo chown ' . $self->userGet() . " ${strFileName}*");
             }
         }
+    }
     else
     {
         confess &log(ASSERT, "There is no original data cached for $strFileName. infoMunge must be called first.");
@@ -1239,6 +1274,7 @@ sub backupDestination {return shift->{strBackupDestination}}
 sub backrestExe {return testRunGet()->backrestExe()}
 sub hardLink {return shift->{bHardLink}}
 sub hasLink {storageRepo()->driver()->className() eq STORAGE_POSIX_DRIVER}
+sub isFS {storageRepo()->driver()->className() ne STORAGE_S3_DRIVER}
 sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()}
 sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER}
 sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY}

test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm

@@ -74,6 +74,7 @@ sub new
             strBackupDestination => $$oParam{strBackupDestination},
             oLogTest => $$oParam{oLogTest},
             bSynthetic => $$oParam{bSynthetic},
+            bRepoLocal => $oParam->{bRepoLocal},
         });
     bless $self, $class;
 

test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm

@@ -60,6 +60,7 @@ sub new
             oLogTest => $$oParam{oLogTest},
             bSynthetic => true,
             bStandby => $$oParam{bStandby},
+            bRepoLocal => $oParam->{bRepoLocal},
         });
     bless $self, $class;
 

test/lib/pgBackRestTest/Env/Host/HostDbTest.pm

@@ -74,6 +74,7 @@ sub new
             strBackupDestination => $$oParam{strBackupDestination},
             oLogTest => $$oParam{oLogTest},
             bStandby => $$oParam{bStandby},
+            bRepoLocal => $oParam->{bRepoLocal},
         });
     bless $self, $class;
 

test/lib/pgBackRestTest/Env/Host/HostS3Test.pm

@@ -0,0 +1,106 @@
+####################################################################################################################################
+# S3 Test Host
+####################################################################################################################################
+package pgBackRestTest::Env::Host::HostS3Test;
+use parent 'pgBackRestTest::Common::HostTest';
+
+####################################################################################################################################
+# Perl includes
+####################################################################################################################################
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+
+use Exporter qw(import);
+    our @EXPORT = qw();
+use File::Basename qw(dirname);
+use Storable qw(dclone);
+
+use pgBackRest::Backup::Common;
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Ini;
+use pgBackRest::Common::Log;
+use pgBackRest::Config::Config;
+use pgBackRest::Manifest;
+use pgBackRest::Protocol::Storage::Helper;
+use pgBackRest::Version;
+
+use pgBackRestTest::Env::Host::HostBaseTest;
+use pgBackRestTest::Common::ContainerTest;
+use pgBackRestTest::Common::ExecuteTest;
+use pgBackRestTest::Common::HostGroupTest;
+use pgBackRestTest::Common::RunTest;
+
+####################################################################################################################################
+# S3 defaults
+####################################################################################################################################
+use constant HOST_S3_ACCESS_KEY                                     => 'accessKey1';
+    push @EXPORT, qw(HOST_S3_ACCESS_KEY);
+use constant HOST_S3_ACCESS_SECRET_KEY                              => 'verySecretKey1';
+    push @EXPORT, qw(HOST_S3_ACCESS_SECRET_KEY);
+use constant HOST_S3_BUCKET                                         => 'pgbackrest-dev';
+    push @EXPORT, qw(HOST_S3_BUCKET);
+use constant HOST_S3_ENDPOINT                                       => 's3.amazonaws.com';
+    push @EXPORT, qw(HOST_S3_ENDPOINT);
+use constant HOST_S3_REGION                                         => 'us-east-1';
+    push @EXPORT, qw(HOST_S3_REGION);
+
+####################################################################################################################################
+# new
+####################################################################################################################################
+sub new
+{
+    my $class = shift;          # Class name
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->new', \@_,
+        );
+
+    # Create the host
+    my $self = $class->SUPER::new(
+        HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', containerRepo() . ':' . testRunGet()->vm() . '-s3-server',
+        'root', testRunGet()->vm());
+    bless $self, $class;
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'self', value => $self, trace => true}
+    );
+}
+
+####################################################################################################################################
+# executeS3
+####################################################################################################################################
+sub executeS3
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strCommand
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->executeS3', \@_,
+            {name => 'strCommand', trace => true},
+        );
+
+    executeTest(
+        'export PYTHONWARNINGS="ignore" && aws --endpoint-url=https://' . $self->ipGet() .
+        ' s3 --no-verify-ssl ' . $strCommand);
+
+    # Return from function and log return values if any
+    return logDebugReturn($strOperation);
+}
+
+1;

test/lib/pgBackRestTest/Env/HostEnvTest.pm

@@ -24,6 +24,9 @@ use pgBackRestTest::Env::Host::HostBaseTest;
 use pgBackRestTest::Env::Host::HostDbCommonTest;
 use pgBackRestTest::Env::Host::HostDbTest;
 use pgBackRestTest::Env::Host::HostDbSyntheticTest;
+use pgBackRestTest::Env::Host::HostS3Test;
+use pgBackRestTest::Common::ContainerTest;
+use pgBackRestTest::Common::ExecuteTest;
 use pgBackRestTest::Common::HostGroupTest;
 use pgBackRestTest::Common::RunTest;
 
@@ -57,6 +60,15 @@ sub setup
     my $oLogTest = shift;
     my $oConfigParam = shift;
 
+    # Start S3 server first since it takes the longest
+    #-------------------------------------------------------------------------------------------------------------------------------
+    my $oHostS3;
+
+    if ($oConfigParam->{bS3})
+    {
+        $oHostS3 = new pgBackRestTest::Env::Host::HostS3Test();
+    }
+
     # Get host group
     my $oHostGroup = hostGroupGet();
 
@@ -70,7 +82,8 @@ sub setup
         $strBackupDestination = defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_BACKUP;
 
         $oHostBackup = new pgBackRestTest::Env::Host::HostBackupTest(
-            {strBackupDestination => $strBackupDestination, bSynthetic => $bSynthetic, oLogTest => $oLogTest});
+            {strBackupDestination => $strBackupDestination, bSynthetic => $bSynthetic, oLogTest => $oLogTest,
+                bRepoLocal => !$oConfigParam->{bS3}});
         $oHostGroup->hostAdd($oHostBackup);
     }
     else
@@ -85,12 +98,12 @@ sub setup
     if ($bSynthetic)
     {
         $oHostDbMaster = new pgBackRestTest::Env::Host::HostDbSyntheticTest(
-            {strBackupDestination => $strBackupDestination, oLogTest => $oLogTest});
+            {strBackupDestination => $strBackupDestination, oLogTest => $oLogTest, bRepoLocal => !$oConfigParam->{bS3}});
     }
     else
     {
         $oHostDbMaster = new pgBackRestTest::Env::Host::HostDbTest(
-            {strBackupDestination => $strBackupDestination, oLogTest => $oLogTest});
+            {strBackupDestination => $strBackupDestination, oLogTest => $oLogTest, bRepoLocal => !$oConfigParam->{bS3}});
     }
 
     $oHostGroup->hostAdd($oHostDbMaster);
@@ -101,24 +114,38 @@ sub setup
     if (defined($$oConfigParam{bStandby}) && $$oConfigParam{bStandby})
     {
         $oHostDbStandby = new pgBackRestTest::Env::Host::HostDbTest(
-            {strBackupDestination => $strBackupDestination, bStandby => true, oLogTest => $oLogTest});
+            {strBackupDestination => $strBackupDestination, bStandby => true, oLogTest => $oLogTest,
+                bRepoLocal => !$oConfigParam->{bS3}});
 
         $oHostGroup->hostAdd($oHostDbStandby);
     }
 
+    # Finalize S3 server
+    #-------------------------------------------------------------------------------------------------------------------------------
+    if (defined($oHostS3))
+    {
+        $oHostGroup->hostAdd($oHostS3, {rstryHostName => ['pgbackrest-dev.s3.amazonaws.com', 's3.amazonaws.com']});
+
+        # Wait for server to start
+        executeTest('docker logs -f ' . $oHostS3->container() . " | grep -m 1 \"server started\"");
+
+        $oHostS3->executeS3('mb s3://' . HOST_S3_BUCKET);
+    }
     # Create db master config
     $oHostDbMaster->configCreate({
         strBackupSource => $$oConfigParam{strBackupSource},
         bCompress => $$oConfigParam{bCompress},
         bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink},
-        bArchiveAsync => $$oConfigParam{bArchiveAsync}});
+        bArchiveAsync => $$oConfigParam{bArchiveAsync},
+        bS3 => $$oConfigParam{bS3}});
 
     # Create backup config if backup host exists
     if (defined($oHostBackup))
     {
         $oHostBackup->configCreate({
             bCompress => $$oConfigParam{bCompress},
-            bHardlink => $$oConfigParam{bHardLink}});
+            bHardlink => $$oConfigParam{bHardLink},
+            bS3 => $$oConfigParam{bS3}});
     }
     # If backup host is not defined set it to db-master
     else
@@ -141,9 +168,23 @@ sub setup
     $self->optionSetTest($oOption, OPTION_DB_PATH, $oHostDbMaster->dbBasePath());
     $self->optionSetTest($oOption, OPTION_REPO_PATH, $oHostBackup->repoPath());
     $self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
+
+    # Set S3 options
+    if (defined($oHostS3))
+    {
+        $self->optionSetTest($oOption, OPTION_REPO_TYPE, REPO_TYPE_S3);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_KEY, HOST_S3_ACCESS_KEY);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_BUCKET, HOST_S3_BUCKET);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_REGION, HOST_S3_REGION);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_HOST, $oHostS3->ipGet());
+        $self->optionBoolSetTest($oOption, OPTION_REPO_S3_VERIFY_SSL, false);
+    }
+
     $self->testResult(sub {$self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH)}, '', 'config load');
 
-    return $oHostDbMaster, $oHostDbStandby, $oHostBackup;
+    return $oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3;
 }
 
 ####################################################################################################################################

test/lib/pgBackRestTest/Env/S3EnvTest.pm

@@ -0,0 +1,56 @@
+####################################################################################################################################
+# S3 Test Environment
+####################################################################################################################################
+package pgBackRestTest::Env::S3EnvTest;
+use parent 'pgBackRestTest::Common::RunTest';
+
+####################################################################################################################################
+# Perl includes
+####################################################################################################################################
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use pgBackRest::Common::Log;
+use pgBackRest::Common::String;
+use pgBackRest::Storage::S3::Driver;
+
+use pgBackRestTest::Common::ExecuteTest;
+use pgBackRestTest::Common::RunTest;
+
+####################################################################################################################################
+# initS3
+####################################################################################################################################
+sub initS3
+{
+    my $self = shift;
+
+    my ($strBucket, $strEndPoint, $strRegion, $strAccessKeyId, $strSecretAccessKey) =
+        ('pgbackrest-dev', 's3.amazonaws.com', 'us-east-1', 'accessKey1', 'verySecretKey1');
+
+    my $strS3ServerPath = $self->testPath() . '/s3server';
+    my $strS3ServerDataPath = "${strS3ServerPath}/data";
+    my $strS3ServerMetaPath = "${strS3ServerPath}/meta";
+    my $strS3ServerLogFile = "${strS3ServerPath}/server.log";
+    storageTest()->pathCreate($strS3ServerDataPath, {bCreateParent => true});
+    storageTest()->pathCreate($strS3ServerMetaPath, {bCreateParent => true});
+
+    $self->{strS3Command} = 'export PYTHONWARNINGS="ignore" && aws s3 --no-verify-ssl';
+
+    executeTest("echo '127.0.0.1 ${strBucket}.${strEndPoint} ${strEndPoint}' | sudo tee -a /etc/hosts");
+    executeTest('sudo sed -i "s/logLevel\"\: \"info\"/logLevel\"\: \"trace\"/" /root/scalitys3/config.json');
+    executeTest("sudo npm start --prefix /root/scalitys3 > ${strS3ServerLogFile} 2>&1 &");
+    executeTest("tail -f ${strS3ServerLogFile} | grep -m 1 \"server started\"");
+    executeTest("$self->{strS3Command} mb s3://pgbackrest-dev");
+
+    # Test variables
+    my $strFile = 'file.txt';
+    my $strFileContent = 'TESTDATA';
+
+    # Initialize the driver
+    return new pgBackRest::Storage::S3::Driver(
+        $strBucket, $strEndPoint, $strRegion, $strAccessKeyId, $strSecretAccessKey, {bVerifySsl => false, lBufferMax => 1048576});
+}
+
+1;

test/lib/pgBackRestTest/Module/Archive/ArchiveGetTest.pm

@@ -42,18 +42,20 @@ sub run
     my $iArchiveMax = 3;
     my $strArchiveTestFile = $self->dataPath() . '/backup.wal2_' . WAL_VERSION_94 . '.bin';
 
-    for (my $bRemote = false; $bRemote <= true; $bRemote++)
+    foreach my $bS3 (false, true)
     {
-    for (my $bCompress = false; $bCompress <= true; $bCompress++)
+    foreach my $bRemote ($bS3 ? (true) : (false, true))
     {
-    for (my $bExists = false; $bExists <= true; $bExists++)
+    foreach my $bCompress ($bS3 ? (true) : (false, true))
+    {
+    foreach my $bExists ($bS3 ? (true) : (false, true))
     {
         # Increment the run, log, and decide whether this unit test should be run
-        if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, exists ${bExists}")) {next}
+        if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, exists ${bExists}, s3 ${bS3}")) {next}
 
         # Create hosts and config
-        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
-            true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress});
+        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
+            true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bS3 => $bS3});
 
         # Create the xlog path
         my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
@@ -175,6 +177,7 @@ sub run
     }
     }
     }
+    }
 }
 
 1;

test/lib/pgBackRestTest/Module/Archive/ArchivePushTest.pm

@@ -80,18 +80,22 @@ sub run
     my $strArchiveChecksum = '72b9da071c13957fb4ca31f05dbd5c644297c2f7';
     my $iArchiveMax = 3;
 
-    for (my $bRemote = false; $bRemote <= true; $bRemote++)
+    foreach my $bS3 (false, true)
     {
-    for (my $bCompress = false; $bCompress <= true; $bCompress++)
+    foreach my $bRemote ($bS3 ? (true) : (false, true))
     {
-    for (my $bArchiveAsync = false; $bArchiveAsync <= true; $bArchiveAsync++)
+    foreach my $bCompress ($bS3 ? (true) : (false, true))
+    {
+    foreach my $bArchiveAsync ($bS3 ? (true) : (false, true))
     {
         # Increment the run, log, and decide whether this unit test should be run
-        if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, arc_async ${bArchiveAsync}", $self->processMax() == 1)) {next}
+        if (!$self->begin(
+            "rmt ${bRemote}, cmp ${bCompress}, arc_async ${bArchiveAsync}, s3 ${bS3}", $self->processMax() == 1)) {next}
 
         # Create hosts, file object, and config
         my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
-            true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => $bArchiveAsync});
+            true, $self->expect(),
+            {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => $bArchiveAsync, bS3 => $bS3});
 
         # Create the xlog path
         my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
@@ -151,10 +155,10 @@ sub run
                            ', archive ' .sprintf('%02x', $iArchive) .
                            " - ${strArchiveFile}");
 
-                # Create a temp file to make sure it is deleted later
+                # Create a temp file to make sure it is deleted later (skip when S3 since it doesn't use temp files)
                 my $strArchiveTmp;
 
-                if ($iBackup == 1 && $iArchive == 2)
+                if (!$bS3 && $iBackup == 1 && $iArchive == 2)
                 {
                     # Should succeed when temp file already exists
                     &log(INFO, '        test archive when tmp file exists');
@@ -344,6 +348,7 @@ sub run
     }
     }
     }
+    }
 }
 
 1;

test/lib/pgBackRestTest/Module/Archive/ArchiveStopTest.pm

@@ -38,18 +38,20 @@ sub run
 
     my $strArchiveTestFile = $self->dataPath() . '/backup.wal2_' . WAL_VERSION_94 . '.bin';
 
-    for (my $bRemote = false; $bRemote <= true; $bRemote++)
+    foreach my $bS3 (false, true)
     {
-    for (my $bCompress = false; $bCompress <= true; $bCompress++)
+    foreach my $bRemote ($bS3 ? (true) : (false, true))
     {
-    for (my $iError = 0; $iError <= $bRemote; $iError++)
+    foreach my $bCompress ($bS3 ? (false) : (false, true))
+    {
+    foreach my $iError ($bS3 ? (1) : ($bRemote ? (0, 1) : (0)))
     {
         # Increment the run, log, and decide whether this unit test should be run
-        if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, error " . ($iError ? 'connect' : 'version'))) {next}
+        if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, error " . ($iError ? 'connect' : 'version') . ", s3 ${bS3}")) {next}
 
         # Create hosts, file object, and config
-        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
-            true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true});
+        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
+            true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bArchiveAsync => true, bS3 => $bS3});
 
         my $oStorage = storageRepo();
 
@@ -117,6 +119,7 @@ sub run
     }
     }
     }
+    }
 }
 
 1;

test/lib/pgBackRestTest/Module/Expire/ExpireExpireTest.pm

@@ -29,6 +29,7 @@ use pgBackRest::Protocol::Storage::Helper;
 use pgBackRestTest::Common::ExecuteTest;
 use pgBackRestTest::Common::RunTest;
 use pgBackRestTest::Env::ExpireEnvTest;
+use pgBackRestTest::Env::Host::HostS3Test;
 use pgBackRestTest::Env::HostEnvTest;
 
 ####################################################################################################################################
@@ -40,6 +41,7 @@ sub initStanzaOption
     my $oOption = shift;
     my $strDbBasePath = shift;
     my $strRepoPath = shift;
+    my $oHostS3 = shift;
 
     $self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
     $self->optionSetTest($oOption, OPTION_DB_PATH, $strDbBasePath);
@@ -50,6 +52,18 @@ sub initStanzaOption
 
     $self->optionSetTest($oOption, OPTION_DB_TIMEOUT, 5);
     $self->optionSetTest($oOption, OPTION_PROTOCOL_TIMEOUT, 6);
+
+    if (defined($oHostS3))
+    {
+        $self->optionSetTest($oOption, OPTION_REPO_TYPE, REPO_TYPE_S3);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_KEY, HOST_S3_ACCESS_KEY);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_KEY_SECRET, HOST_S3_ACCESS_SECRET_KEY);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_BUCKET, HOST_S3_BUCKET);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_ENDPOINT, HOST_S3_ENDPOINT);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_REGION, HOST_S3_REGION);
+        $self->optionSetTest($oOption, OPTION_REPO_S3_HOST, $oHostS3->ipGet());
+        $self->optionBoolSetTest($oOption, OPTION_REPO_S3_VERIFY_SSL, false);
+    }
 }
 
 ####################################################################################################################################
@@ -64,12 +78,14 @@ sub run
     my $strDescription;
     my $oOption = {};
 
+    my $bS3 = false;
+
     if ($self->begin("local"))
     {
         # Create hosts, file object, and config
-        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(true, $self->expect());
+        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(true, $self->expect(), {bS3 => $bS3});
 
-        $self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath});
+        $self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3);
         $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE);
 
         # Create the test object
@@ -164,9 +180,9 @@ sub run
     if ($self->begin("Expire::stanzaUpgrade"))
     {
         # Create hosts, file object, and config
-        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(true, $self->expect());
+        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(true, $self->expect(), {bS3 => $bS3});
 
-        $self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath});
+        $self->initStanzaOption($oOption, $oHostDbMaster->dbBasePath(), $oHostBackup->{strRepoPath}, $oHostS3);
         $self->configLoadExpect(dclone($oOption), CMD_STANZA_CREATE);
 
         # Create the test object

test/lib/pgBackRestTest/Module/Full/FullRealTest.pm

@@ -43,20 +43,23 @@ sub run
 {
     my $self = shift;
 
-    foreach my $bHostBackup (false, true)
+    foreach my $bS3 (false, true)
     {
-    foreach my $bHostStandby (false, true)
+    foreach my $bHostBackup ($bS3 ? (true) : (false, true))
+    {
+    foreach my $bHostStandby ($bS3 ? (false) : (false, true))
     {
     foreach my $strBackupDestination (
-        $bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER))
+        $bS3 || $bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER))
     {
-    foreach my $bArchiveAsync ($bHostStandby ? (false) : (false, true))
+    foreach my $bArchiveAsync ($bS3 ? (true) : ($bHostStandby ? (false) : (false, true)))
     {
-    foreach my $bCompress ($bHostStandby ? (false) : (false, true))
+    foreach my $bCompress ($bS3 ? (true) : ($bHostStandby ? (false) : (false, true)))
     {
         # Increment the run, log, and decide whether this unit test should be run
         next if (!$self->begin(
-            "bkp ${bHostBackup}, sby ${bHostStandby}, dst ${strBackupDestination}, asy ${bArchiveAsync}, cmp ${bCompress}",
+            "bkp ${bHostBackup}, sby ${bHostStandby}, dst ${strBackupDestination}, asy ${bArchiveAsync}, cmp ${bCompress}," .
+                " s3 ${bS3}",
             $self->processMax() == 1 && $self->pgVersion() eq PG_VERSION_95));
 
         if ($bHostStandby && $self->pgVersion() < PG_VERSION_HOT_STANDBY)
@@ -66,13 +69,13 @@ sub run
         }
 
         # Create hosts, file object, and config
-        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
+        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
             false, $self->expect(),
             {bHostBackup => $bHostBackup, bStandby => $bHostStandby, strBackupDestination => $strBackupDestination,
-             bCompress => $bCompress, bArchiveAsync => $bArchiveAsync});
+             bCompress => $bCompress, bArchiveAsync => $bArchiveAsync, bS3 => $bS3});
 
         # Determine if extra tests are performed.  Extra tests should not be primary tests for compression or async archiving.
-        my $bTestExtra = ($self->runCurrent() == 1) || ($self->runCurrent() == 7);
+        my $bTestExtra = $self->runCurrent() == 1 || $self->runCurrent() == 7 || $self->runCurrent() == 12;
 
         $oHostDbMaster->clusterCreate();
 
@@ -251,8 +254,11 @@ sub run
             forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE);
             forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE . INI_COPY_EXT);
 
-            $oHostBackup->stanzaCreate('fail on backup info file missing from non-empty dir',
-                {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
+            if (!$bS3)
+            {
+                $oHostBackup->stanzaCreate('fail on backup info file missing from non-empty dir',
+                    {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
+            }
 
             # Force the backup.info file to be recreated
             $oHostBackup->stanzaCreate('verify success with force', {strOptionalParam => ' --' . OPTION_FORCE});
@@ -899,6 +905,7 @@ sub run
     }
     }
     }
+    }
 }
 
 1;

test/lib/pgBackRestTest/Module/Full/FullSyntheticTest.pm

@@ -29,11 +29,12 @@ use pgBackRest::Protocol::Storage::Helper;
 use pgBackRest::Version;
 
 use pgBackRestTest::Common::ContainerTest;
-use pgBackRestTest::Env::HostEnvTest;
 use pgBackRestTest::Common::ExecuteTest;
 use pgBackRestTest::Common::FileTest;
 use pgBackRestTest::Common::RunTest;
 use pgBackRestTest::Env::Host::HostBackupTest;
+use pgBackRestTest::Env::Host::HostS3Test;
+use pgBackRestTest::Env::HostEnvTest;
 
 ####################################################################################################################################
 # Build PostgreSQL pages for testing
@@ -58,18 +59,20 @@ sub run
 {
     my $self = shift;
 
-    for (my $bRemote = false; $bRemote <= true; $bRemote++)
+    foreach my $bS3 (false, true)
     {
-    for (my $bCompress = false; $bCompress <= true; $bCompress++)
+    foreach my $bRemote ($bS3 ? (true) : (false, true))
     {
-    for (my $bHardLink = false; $bHardLink <= true; $bHardLink++)
+    foreach my $bCompress ($bS3 ? (false) : (false, true))
+    {
+    foreach my $bHardLink ($bS3 ? (false) : (false, true))
     {
         # Increment the run, log, and decide whether this unit test should be run
-        if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, hardlink ${bHardLink}", $self->processMax() == 1)) {next}
+        if (!$self->begin("rmt ${bRemote}, cmp ${bCompress}, hardlink ${bHardLink}, s3 ${bS3}", $self->processMax() == 1)) {next}
 
         # Create hosts, file object, and config
-        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
-            true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bHardLink => $bHardLink});
+        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
+            true, $self->expect(), {bHostBackup => $bRemote, bCompress => $bCompress, bHardLink => $bHardLink, bS3 => $bS3});
 
         # Determine if this is a neutral test, i.e. we only want to do it once for local and once for remote.  Neutral means
         # that options such as compression and hardlinks are disabled
@@ -316,7 +319,7 @@ sub run
             {oExpectedManifest => \%oManifest,
                 strOptionalParam => $strOptionalParam . ($bRemote ? ' --cmd-ssh=/usr/bin/ssh' : '') . ' --' . OPTION_BUFFER_SIZE .
                     '=16384 --' . OPTION_CHECKSUM_PAGE,
-                strRepoType => REPO_TYPE_CIFS, strTest => $strTestPoint, fTestDelay => 0});
+                strRepoType => $bS3 ? undef : REPO_TYPE_CIFS, strTest => $strTestPoint, fTestDelay => 0});
 
         # Error on backup option to check logging
         #-----------------------------------------------------------------------------------------------------------------------
@@ -435,9 +438,17 @@ sub run
         forceStorageRemove(storageRepo(), "${strResumePath}/" . FILE_MANIFEST);
 
         # Create a temp file in backup temp root to be sure it's deleted correctly
-        executeTest("sudo touch ${strResumePath}/file.tmp" . ($bCompress ? '.gz' : ''),
-                    {bRemote => $bRemote});
-        executeTest("sudo chown " . BACKREST_USER . " ${strResumePath}/file.tmp" . ($bCompress ? '.gz' : ''));
+        my $strTempFile = "${strResumePath}/file.tmp" . ($bCompress ? '.gz' : '');
+
+        if ($bS3)
+        {
+            $oHostS3->executeS3('cp /etc/hosts s3://' . HOST_S3_BUCKET . "${strTempFile}");
+        }
+        else
+        {
+            executeTest("sudo touch ${strTempFile}", {bRemote => $bRemote});
+            executeTest("sudo chown " . BACKREST_USER . " ${strResumePath}/file.tmp" . ($bCompress ? '.gz' : ''));
+        }
 
         $strFullBackup = $oHostBackup->backup(
             $strType, 'resume',
@@ -454,7 +465,7 @@ sub run
             $oHostBackup->backup(
                 $strType, 'invalid repo',
                 {oExpectedManifest => \%oManifest, strOptionalParam => '--' . OPTION_REPO_PATH . '=/bogus_path' .
-                 '  --log-level-console=detail', iExpectedExitStatus => ERROR_PATH_MISSING});
+                 '  --log-level-console=detail', iExpectedExitStatus => $bS3 ? ERROR_FILE_MISSING : ERROR_PATH_MISSING});
         }
 
         # Restore - tests various mode, extra files/paths, missing files/paths
@@ -1185,6 +1196,7 @@ sub run
     }
     }
     }
+    }
 }
 
 1;

test/lib/pgBackRestTest/Module/Stanza/StanzaCreateTest.pm

@@ -40,14 +40,16 @@ sub run
 {
     my $self = shift;
 
-    foreach my $bRemote (false, true)
+    foreach my $bS3 (false, true)
+    {
+    foreach my $bRemote ($bS3 ? (false) : (false, true))
     {
         # Increment the run, log, and decide whether this unit test should be run
-        if (!$self->begin("remote ${bRemote}")) {next}
+        if (!$self->begin("remote $bRemote, s3 $bS3")) {next}
 
         # Create hosts, file object, and config
-        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
-            true, $self->expect(), {bHostBackup => $bRemote});
+        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
+            true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3});
 
         # Create the stanza
         $oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_OPEN,
@@ -83,21 +85,25 @@ sub run
         $oHostBackup->stanzaCreate('fail on archive info file missing from non-empty dir',
             {iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . OPTION_ONLINE});
 
-        # Change the permissions of the archive file so it cannot be read
-        forceStorageMode(
-            storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' .
-                COMPRESS_EXT,
-            '220');
+        # S3 doesn't support filesystem-style permissions so skip these tests
+        if (!$bS3)
+        {
+            # Change the permissions of the archive file so it cannot be read
+            forceStorageMode(
+                storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' .
+                    COMPRESS_EXT,
+                '220');
 
-        # Force creation of the info file but fail on gunzip
-        $oHostBackup->stanzaCreate('gunzip fail on forced stanza-create',
-            {iExpectedExitStatus => ERROR_FILE_OPEN, strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
+            # Force creation of the info file but fail on gunzip
+            $oHostBackup->stanzaCreate('gunzip fail on forced stanza-create',
+                {iExpectedExitStatus => ERROR_FILE_OPEN, strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
 
-        # Change permissions back
-        forceStorageMode(
-            storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' .
-                COMPRESS_EXT,
-            '640');
+            # Change permissions back
+            forceStorageMode(
+                storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_94 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' .
+                    COMPRESS_EXT,
+                '640');
+        }
 
         # Force creation of archive info from the gz file
         $oHostBackup->stanzaCreate('force create archive.info from gz file',
@@ -170,6 +176,7 @@ sub run
         $oHostBackup->stanzaCreate('force with missing WAL archive directory',
             {strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE});
     }
+    }
 }
 
 1;

test/lib/pgBackRestTest/Module/Stanza/StanzaUpgradeTest.pm

@@ -43,14 +43,16 @@ sub run
 {
     my $self = shift;
 
-    for (my $bRemote = false; $bRemote <= true; $bRemote++)
+    foreach my $bS3 (false, true)
+    {
+    foreach my $bRemote ($bS3 ? (true) : (false, true))
     {
         # Increment the run, log, and decide whether this unit test should be run
-        if (!$self->begin($bRemote ? "remote" : "local")) {next}
+        if (!$self->begin("remote $bRemote, s3 $bS3")) {next}
 
         # Create hosts, file object, and config
-        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup) = $self->setup(
-            true, $self->expect(), {bHostBackup => $bRemote});
+        my ($oHostDbMaster, $oHostDbStandby, $oHostBackup, $oHostS3) = $self->setup(
+            true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3});
 
         # Create the test path for pg_control
         storageDb()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
@@ -169,6 +171,7 @@ sub run
         # Confirm info command displays the JSON correctly
         $oHostDbMaster->info('db upgraded - db-1 and db-2 listed', {strOutput => INFO_OUTPUT_JSON});
     }
+    }
 }
 
 1;

test/lib/pgBackRestTest/Module/Storage/StorageS3AuthTest.pm

@@ -0,0 +1,111 @@
+####################################################################################################################################
+# S3 Authentication Tests
+####################################################################################################################################
+package pgBackRestTest::Module::Storage::StorageS3AuthTest;
+use parent 'pgBackRestTest::Common::RunTest';
+
+####################################################################################################################################
+# Perl includes
+####################################################################################################################################
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use POSIX qw(strftime);
+
+use pgBackRest::Common::Exception;
+use pgBackRest::Common::Log;
+use pgBackRest::Common::Wait;
+use pgBackRest::Storage::S3::Auth;
+
+####################################################################################################################################
+# run
+####################################################################################################################################
+sub run
+{
+    my $self = shift;
+
+    ################################################################################################################################
+    if ($self->begin('s3DateTime'))
+    {
+        $self->testResult(sub {s3DateTime(1492127085)}, '20170413T234445Z', 'format date/time');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        waitRemainder();
+        $self->testResult(sub {s3DateTime()}, strftime("%Y%m%dT%k%M%SZ", gmtime()), 'format current date/time');
+    }
+
+    ################################################################################################################################
+    if ($self->begin('s3CanonicalRequest'))
+    {
+        $self->testResult(
+            sub {s3CanonicalRequest(
+                'GET', qw(/), 'list-type=2',
+                {'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z',
+                    'x-amz-content-sha256' => '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9'},
+                '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')},
+            "(GET\n/\nlist-type=2\nhost:bucket.s3.amazonaws.com\n" .
+                "x-amz-content-sha256:705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9\n" .
+                "x-amz-date:20170606T121212Z\n\nhost;x-amz-content-sha256;x-amz-date\n" .
+                '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9' .
+                ', host;x-amz-content-sha256;x-amz-date)',
+            'canonical request');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        $self->testException(
+            sub {s3CanonicalRequest(
+                'GET', qw(/), 'list-type=2', {'Host' => 'bucket.s3.amazonaws.com'},
+                '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')},
+            ERROR_ASSERT, "header 'Host' must be lower case");
+    }
+
+    ################################################################################################################################
+    if ($self->begin('s3SigningKey'))
+    {
+        $self->testResult(
+            sub {unpack('H*', s3SigningKey('20170412', 'us-east-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))},
+            '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9', 'signing key');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        $self->testResult(
+            sub {unpack('H*', s3SigningKey('20170412', 'us-east-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))},
+            '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9', 'same signing key from cache');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        $self->testResult(
+            sub {unpack('H*', s3SigningKey('20170505', 'us-west-1', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'))},
+            'c1a1cb590bbc38ba789c8e5695a1ec0cd7fd44c6949f922e149005a221524c09', 'new signing key');
+    }
+
+    ################################################################################################################################
+    if ($self->begin('s3StringToSign'))
+    {
+        $self->testResult(
+            sub {s3StringToSign(
+                '20170412T141414Z', 'us-east-1', '705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9')},
+            "AWS4-HMAC-SHA256\n20170412T141414Z\n20170412/us-east-1/s3/aws4_request\n" .
+                "705636ecdedffc09f140497bcac3be1e8d069008ecc6a8029e104d6291b4e4e9",
+            'string to sign');
+    }
+
+    ################################################################################################################################
+    if ($self->begin('s3AuthorizationHeader'))
+    {
+        $self->testResult(
+            sub {s3AuthorizationHeader(
+                'us-east-1', 'bucket.s3.amazonaws.com', 'GET', qw(/), 'list-type=2', '20170606T121212Z',
+                {'host' => 'bucket.s3.amazonaws.com', 'x-amz-date' => '20170606T121212Z'},
+                'AKIAIOSFODNN7EXAMPLE', 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+                'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')},
+            '{authorization => AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,' .
+                'SignedHeaders=host;x-amz-content-sha256;x-amz-date,' .
+                'Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3,' .
+                ' host => bucket.s3.amazonaws.com,' .
+                ' x-amz-content-sha256 => e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855,' .
+                ' x-amz-date => 20170606T121212Z}',
+            'authorization header request');
+    }
+}
+
+1;

test/lib/pgBackRestTest/Module/Storage/StorageS3Test.pm

@@ -0,0 +1,208 @@
+####################################################################################################################################
+# S3 Storage Tests
+####################################################################################################################################
+package pgBackRestTest::Module::Storage::StorageS3Test;
+use parent 'pgBackRestTest::Env::S3EnvTest';
+
+####################################################################################################################################
+# Perl includes
+####################################################################################################################################
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+use Digest::SHA qw(sha1_hex);
+
+use pgBackRest::Common::Log;
+use pgBackRest::Common::String;
+use pgBackRest::Storage::S3::Driver;
+
+use pgBackRestTest::Common::ExecuteTest;
+use pgBackRestTest::Common::RunTest;
+
+####################################################################################################################################
+# initTest
+####################################################################################################################################
+sub initTest
+{
+    my $self = shift;
+
+    executeTest("$self->{strS3Command} rm --recursive s3://pgbackrest-dev");
+}
+
+####################################################################################################################################
+# run
+####################################################################################################################################
+sub run
+{
+    my $self = shift;
+
+    # Initialize the driver
+    my $oS3 = $self->initS3();
+    my $oStorage = new pgBackRest::Storage::Local('', $oS3);
+
+    # Test variables
+    my $strFile = 'file.txt';
+    my $strFileContent = 'TESTDATA';
+    my $iFileLength = length($strFileContent);
+
+    ################################################################################################################################
+    if ($self->begin('exists()'))
+    {
+        #---------------------------------------------------------------------------------------------------------------------------
+        $self->testResult(sub {$oStorage->exists($strFile)}, false, 'root file does not exist');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        storageTest()->put($strFile, $strFileContent);
+        executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev");
+
+        $self->testResult(sub {$oStorage->exists($strFile)}, true, 'root file exists');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        $self->testResult(sub {$oStorage->pathExists('/path/to')}, false, 'sub path does not exist');
+        $self->testResult(sub {$oStorage->exists("/path/to/${strFile}")}, false, 'sub file does not exist');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev/path/to/${strFile}");
+
+        $self->testResult(sub {$oStorage->pathExists('/path/to')}, true, 'sub path exists');
+        # $oStorage->pathExists('/path/to');
+        $self->testResult(sub {$oStorage->exists("/path/to/${strFile}")}, true, 'sub file exists');
+    }
+
+    ################################################################################################################################
+    if ($self->begin('manifest()'))
+    {
+        #---------------------------------------------------------------------------------------------------------------------------
+        $self->testResult(sub {$oStorage->manifest('')}, '{. => {type => d}}', 'no files');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        storageTest()->put($strFile, $strFileContent);
+        storageTest()->put("${strFile}2", $strFileContent . '2');
+
+        executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev");
+        executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2");
+
+        $self->testResult(
+            sub {$oStorage->manifest('')},
+            '{. => {type => d}, file.txt => {size => 8, type => f}, path => {type => d}, path/to => {type => d},' .
+                ' path/to/file.txt2 => {size => 9, type => f}}',
+            'root path');
+        $self->testResult(
+            sub {$oStorage->manifest('/path/to')}, '{. => {type => d}, file.txt2 => {size => 9, type => f}}', 'sub path');
+    }
+
+    ################################################################################################################################
+    if ($self->begin('list()'))
+    {
+        #---------------------------------------------------------------------------------------------------------------------------
+        $self->testResult(
+            sub {$oStorage->list('')}, '[undef]', 'no files');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        storageTest()->put($strFile, $strFileContent);
+        storageTest()->put("${strFile}2", $strFileContent . '2');
+
+        executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev");
+        executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2");
+
+        $self->testResult(sub {$oStorage->list('')}, '(file.txt, path)', 'root path');
+        $self->testResult(sub {$oStorage->list('/path/to')}, 'file.txt2', 'sub path');
+    }
+
+    ################################################################################################################################
+    if ($self->begin('remove()'))
+    {
+        #---------------------------------------------------------------------------------------------------------------------------
+        $oStorage->put($strFile, $strFileContent);
+        $oStorage->put("/path/to/${strFile}2", $strFileContent);
+        $oStorage->put("/path/to/${strFile}3", $strFileContent);
+        $oStorage->put("/path/to/${strFile}4", $strFileContent);
+
+        $self->testResult(
+            sub {$oStorage->manifest('/')},
+            '{. => {type => d}, file.txt => {size => 8, type => f}, path => {type => d}, path/to => {type => d},' .
+                ' path/to/file.txt2 => {size => 8, type => f}, path/to/file.txt3 => {size => 8, type => f},' .
+                ' path/to/file.txt4 => {size => 8, type => f}}',
+            'check manifest');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        $self->testResult(sub {$oStorage->remove('/path/to', {bRecurse => true})}, true, 'remove subpath');
+
+        $self->testResult(
+            sub {$oStorage->manifest('/')},
+            '{. => {type => d}, file.txt => {size => 8, type => f}}', 'check manifest');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        $self->testResult(sub {$oStorage->remove($strFile)}, true, 'remove file');
+
+        $self->testResult(sub {$oStorage->manifest('/')}, '{. => {type => d}}', 'check manifest');
+    }
+
+    ################################################################################################################################
+    if ($self->begin('info()'))
+    {
+        #---------------------------------------------------------------------------------------------------------------------------
+        storageTest()->put($strFile, $strFileContent);
+        storageTest()->put("${strFile}2", $strFileContent . '2');
+
+        executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile} s3://pgbackrest-dev");
+        executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev");
+        executeTest("$self->{strS3Command} cp " . $self->testPath() . "/${strFile}2 s3://pgbackrest-dev/path/to/${strFile}2");
+
+        $self->testResult(sub {$oStorage->info($strFile)->size()}, 8, 'file size');
+        $self->testResult(sub {$oStorage->info("/path/to/${strFile}2")->size()}, 9, 'file 2 size');
+    }
+
+    ################################################################################################################################
+    if ($self->begin('openRead() && S3::FileRead'))
+    {
+        # Create a random 1mb file
+        my $strRandomFile = $self->testPath() . '/random1mb.bin';
+        executeTest("dd if=/dev/urandom of=${strRandomFile} bs=1024k count=1", {bSuppressStdErr => true});
+        my $strRandom = ${storageTest()->get($strRandomFile)};
+
+        executeTest("$self->{strS3Command} cp ${strRandomFile} s3://pgbackrest-dev/path/to/${strFile}");
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        my $tBuffer;
+        my $oFileRead = $self->testResult(sub {$oS3->openRead("/path/to/${strFile}")}, '[object]', 'open read');
+        $self->testResult(sub {$oFileRead->read(\$tBuffer, 524288)}, 524288, '    read half');
+        $self->testResult(sub {$oFileRead->read(\$tBuffer, 524288)}, 524288, '    read half');
+        $self->testResult(sub {$oFileRead->read(\$tBuffer, 512)}, 0, '    read 0');
+        $self->testResult(length($tBuffer), 1048576, '    check length');
+        $self->testResult(sha1_hex($tBuffer), sha1_hex($strRandom), '    check hash');
+    }
+
+    ################################################################################################################################
+    if ($self->begin('openWrite() S3::FileWrite'))
+    {
+        # Create a random 1mb file
+        my $strRandomFile = $self->testPath() . '/random1mb.bin';
+        executeTest("dd if=/dev/urandom of=${strRandomFile} bs=1024k count=1", {bSuppressStdErr => true});
+        my $strRandom = ${storageTest()->get($strRandomFile)};
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        my $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write');
+        $self->testResult(sub {$oFileWrite->close()}, true, '    close without writing');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write');
+        $self->testResult(sub {$oFileWrite->write()}, 0, '    write undef');
+        $self->testResult(sub {$oFileWrite->write(\$strFileContent)}, $iFileLength, '    write');
+        $self->testResult(sub {$oFileWrite->close()}, true, '    close');
+
+        #---------------------------------------------------------------------------------------------------------------------------
+        $oFileWrite = $self->testResult(sub {$oS3->openWrite("/path/to/${strFile}")}, '[object]', 'open write');
+
+        for (my $iIndex = 1; $iIndex <= 17; $iIndex++)
+        {
+            $self->testResult(sub {$oFileWrite->write(\$strRandom)}, 1024 * 1024, '    write 1mb');
+        }
+
+        $self->testResult(sub {$oFileWrite->close()}, true, '    close');
+    }
+}
+
+1;