diff --git a/doc/lib/BackRestDoc/Common/DocExecute.pm b/doc/lib/BackRestDoc/Common/DocExecute.pm index 196a54701..d7cf9fd3f 100644 --- a/doc/lib/BackRestDoc/Common/DocExecute.pm +++ b/doc/lib/BackRestDoc/Common/DocExecute.pm @@ -630,7 +630,6 @@ sub backrestConfig my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}}); delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_LEVEL_STDERR}); delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_TIMESTAMP}); - delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'}); if (keys(%{$$oConfigClean{&CFGDEF_SECTION_GLOBAL}}) == 0) { @@ -1086,7 +1085,7 @@ sub sectionChildProcess $self->{oManifest}->variableReplace($oChild->paramGet('user')), $$hCacheKey{os}, defined($oChild->paramGet('mount', false)) ? [$self->{oManifest}->variableReplace($oChild->paramGet('mount'))] : undef, - $$hCacheKey{option}, $$hCacheKey{param}); + $$hCacheKey{option}, $$hCacheKey{param}, $$hCacheKey{'update-hosts'}); $self->{host}{$$hCacheKey{name}} = $oHost; $self->{oManifest}->variableSet('host-' . $hCacheKey->{id} . '-ip', $oHost->{strIP}, true); @@ -1094,7 +1093,7 @@ sub sectionChildProcess # Add to the host group my $oHostGroup = hostGroupGet(); - $oHostGroup->hostAdd($oHost, {bUpdateHosts => $$hCacheKey{'update-hosts'}}); + $oHostGroup->hostAdd($oHost); # Execute initialize commands foreach my $oExecute ($oChild->nodeList('execute', false)) diff --git a/doc/resource/fake-cert/.gitignore b/doc/resource/fake-cert/.gitignore new file mode 100644 index 000000000..c80de9a8f --- /dev/null +++ b/doc/resource/fake-cert/.gitignore @@ -0,0 +1,2 @@ +*.csr +*.srl diff --git a/doc/resource/fake-cert/README.md b/doc/resource/fake-cert/README.md new file mode 100644 index 000000000..5e2e08727 --- /dev/null +++ b/doc/resource/fake-cert/README.md @@ -0,0 +1,28 @@ +# pgBackRest Documentation Certificates + +The certificates in this directory are used for documentation generation only and should not be used for actual services. + +## pgBackRest CA + +Generate a CA that will be used to sign documentation certificates. It can be installed in the documentation containers to make certificates signed by it valid. + +``` +cd [pgbackrest-root]/doc/resource/fake-cert + +openssl ecparam -genkey -name prime256v1 | openssl ec -out ca.key +openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 \ + -subj "/C=US/ST=All/L=All/O=pgBackRest/CN=pgbackrest.org" +``` + +## S3 Certificate + +Mimic an S3 certificate for the `us-east-1`/`us-east-2` region to generate S3 documentation. + +``` +cd [pgbackrest-root]/doc/resource/fake-cert + +openssl ecparam -genkey -name prime256v1 | openssl ec -out s3-server.key +openssl req -new -sha256 -nodes -out s3-server.csr -key s3-server.key -config s3.cnf +openssl x509 -req -in s3-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial \ + -out s3-server.crt -days 99999 -extensions v3_req -extfile s3.cnf +``` diff --git a/doc/resource/fake-cert/ca.crt b/doc/resource/fake-cert/ca.crt new file mode 100644 index 000000000..8d9a9dc88 --- /dev/null +++ b/doc/resource/fake-cert/ca.crt @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB+jCCAaCgAwIBAgIJAJDUUhiBUbmEMAoGCCqGSM49BAMCMFcxCzAJBgNVBAYT +AlVTMQwwCgYDVQQIDANBbGwxDDAKBgNVBAcMA0FsbDETMBEGA1UECgwKcGdCYWNr +UmVzdDEXMBUGA1UEAwwOcGdiYWNrcmVzdC5vcmcwIBcNMTkwNTI3MDAxOTU5WhgP +MjI5MzAzMTAwMDE5NTlaMFcxCzAJBgNVBAYTAlVTMQwwCgYDVQQIDANBbGwxDDAK +BgNVBAcMA0FsbDETMBEGA1UECgwKcGdCYWNrUmVzdDEXMBUGA1UEAwwOcGdiYWNr +cmVzdC5vcmcwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQYHUcSknRDL+fgFJZI +IC73Ju75yA0203IxPO35i8mVb9CcWVhEgHmS+cQ6SfY6GC7V61VB7gwzQ+XESi2p +ndhJo1MwUTAdBgNVHQ4EFgQUYMbKIlTUE6gklw8KcSC6fnlOitwwHwYDVR0jBBgw +FoAUYMbKIlTUE6gklw8KcSC6fnlOitwwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjO +PQQDAgNIADBFAiEA1Bzy17/6jQimg3ROZTrVGkRtAuzTtjgDParHFrIhSDoCIH43 +OeOUaPVb0rXGPLu9rFpjPOmtFSW3lf4skheJMKyN +-----END CERTIFICATE----- diff --git a/doc/resource/fake-cert/ca.key b/doc/resource/fake-cert/ca.key new file mode 100644 index 000000000..214bc7fcc --- /dev/null +++ b/doc/resource/fake-cert/ca.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIB5f3SxfiZ92GMpuqpfTiPO3xaVOnxRh6qVAoRtu7NOZoAoGCCqGSM49 +AwEHoUQDQgAEGB1HEpJ0Qy/n4BSWSCAu9ybu+cgNNtNyMTzt+YvJlW/QnFlYRIB5 +kvnEOkn2Ohgu1etVQe4MM0PlxEotqZ3YSQ== +-----END EC PRIVATE KEY----- diff --git a/doc/resource/fake-cert/s3-server.crt b/doc/resource/fake-cert/s3-server.crt new file mode 100644 index 000000000..3f1985a82 --- /dev/null +++ b/doc/resource/fake-cert/s3-server.crt @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICbTCCAhOgAwIBAgIJAODTXyGnxWtVMAoGCCqGSM49BAMCMFcxCzAJBgNVBAYT +AlVTMQwwCgYDVQQIDANBbGwxDDAKBgNVBAcMA0FsbDETMBEGA1UECgwKcGdCYWNr +UmVzdDEXMBUGA1UEAwwOcGdiYWNrcmVzdC5vcmcwIBcNMTkwNTI3MDIwODEwWhgP +MjI5MzAzMTAwMjA4MTBaMIGBMQswCQYDVQQGEwJVUzEMMAoGA1UECAwDQWxsMQww +CgYDVQQHDANBbGwxEzARBgNVBAoMCnBnQmFja1Jlc3QxHDAaBgNVBAsME1VuaXQg +VGVzdGluZyBEb21haW4xIzAhBgNVBAMMGnMzLnVzLWVhc3QtMS5hbWF6b25hd3Mu +Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEe2dO1v1gE0Qj4H407i0K8tN +kASkveckACPFzXs2i/++rZY4bwUub08JcMRv0WWwnRzOoumsN26Ge454vTbjoqOB +mjCBlzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DB9BgNVHREEdjB0ghpzMy51cy1l +YXN0LTEuYW1hem9uYXdzLmNvbYIcKi5zMy51cy1lYXN0LTEuYW1hem9uYXdzLmNv +bYIaczMudXMtZWFzdC0yLmFtYXpvbmF3cy5jb22CHCouczMudXMtZWFzdC0yLmFt +YXpvbmF3cy5jb20wCgYIKoZIzj0EAwIDSAAwRQIgLiE7LuK6O/bKo70XPUi6xoDE +ew+EHO31klTOeWiS6oMCIQCHMEqSAcDF/gnG/UXnp2viHOFjnY+NZgQo76l+/2mE +iQ== +-----END CERTIFICATE----- diff --git a/doc/resource/fake-cert/s3-server.key b/doc/resource/fake-cert/s3-server.key new file mode 100644 index 000000000..15e4e3b49 --- /dev/null +++ b/doc/resource/fake-cert/s3-server.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIBhweMaCuhrRJy6hLV9X7QRCorDdyiUvSWEySHXZJM4DoAoGCCqGSM49 +AwEHoUQDQgAEEe2dO1v1gE0Qj4H407i0K8tNkASkveckACPFzXs2i/++rZY4bwUu +b08JcMRv0WWwnRzOoumsN26Ge454vTbjog== +-----END EC PRIVATE KEY----- diff --git a/doc/resource/fake-cert/s3.cnf b/doc/resource/fake-cert/s3.cnf new file mode 100644 index 000000000..1aaf16ace --- /dev/null +++ b/doc/resource/fake-cert/s3.cnf @@ -0,0 +1,25 @@ +[req] +default_bits = 4096 +prompt = no +default_md = sha256 +req_extensions = v3_req +distinguished_name = dn + +[ dn ] +C=US +ST=All +L=All +O=pgBackRest +OU=Unit Testing Domain +CN = s3.us-east-1.amazonaws.com + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[ alt_names ] +DNS.1 = s3.us-east-1.amazonaws.com +DNS.2 = *.s3.us-east-1.amazonaws.com +DNS.3 = s3.us-east-2.amazonaws.com +DNS.4 = *.s3.us-east-2.amazonaws.com diff --git a/doc/xml/dtd/doc.dtd b/doc/xml/dtd/doc.dtd index 34e35f0a4..34601d794 100644 --- a/doc/xml/dtd/doc.dtd +++ b/doc/xml/dtd/doc.dtd @@ -200,6 +200,7 @@ + diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index 9edf3276d..b927201a5 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -54,6 +54,14 @@ /pgbackrest + + /usr/local/share/ca-certificates + /etc/pki/ca-trust/source/anchors + + + resource/fake-cert + {[pgbackrest-host-repo-path]}/doc/{[fake-cert-path-relative]} + {[os-debian-pg-version]} {[os-centos6-pg-version]} {[os-centos7-pg-version]} @@ -128,6 +136,15 @@ pg_switch_xlog pg_switch_wal + + y + demo-bucket + demo-repo + us-east-1 + s3.{[s3-region]}.amazonaws.com + accessKey1 + verySecretKey1 + pgbackrest/doc:{[os-type]} @@ -228,11 +245,18 @@ echo ' StrictHostKeyChecking no' >> /root/.ssh/config && \ chmod 600 /root/.ssh/* + + wget https://dl.min.io/client/mc/release/linux-amd64/mc -qO /usr/bin/mc && \ + chmod 755 /usr/bin/mc + + COPY {[fake-cert-path-relative]}/ca.crt {[ca-cert-path]}/pgbackrest-ca.crt + {[copy-ca-cert]} + # Fix root tty RUN sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile && \ @@ -241,7 +265,10 @@ # Install base packages RUN apt-get update && \ - apt-get install -y sudo ssh wget vim gnupg lsb-release 2>&1 + apt-get install -y sudo ssh wget vim gnupg lsb-release iputils-ping ca-certificates 2>&1 + + # Install CA certificate + RUN update-ca-certificates # Install PostgreSQL RUN RELEASE_CODENAME=`lsb_release -c | awk '{print $2}'` && \ @@ -261,13 +288,21 @@ RUN adduser --disabled-password --gecos "" {[host-user]} && \ echo '%{[host-user]} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers + RUN {[minio-client-install]} + ENTRYPOINT service ssh restart && bash + {[copy-ca-cert]} + # Install packages RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&1 + # Install CA certificate + RUN update-ca-trust enable && \ + update-ca-trust extract + # Regenerate SSH keys RUN rm -f /etc/ssh/ssh_host_rsa_key* && \ ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key @@ -284,12 +319,16 @@ RUN adduser -n {[host-user]} && \ echo '{[host-user]} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/{[host-user]} + RUN {[minio-client-install]} + ENTRYPOINT /usr/sbin/sshd -D ENV container docker + {[copy-ca-cert]} + RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \ systemd-tmpfiles-setup.service ] || rm -f $i; done); \ rm -f /lib/systemd/system/multi-user.target.wants/*;\ @@ -305,6 +344,9 @@ # Install packages RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&1 + # Install CA certificate + RUN update-ca-trust extract + # Regenerate SSH keys RUN rm -f /etc/ssh/ssh_host_rsa_key* && \ ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key && \ @@ -328,6 +370,8 @@ ln -s /usr/lib/systemd/system/systemd-user-sessions.service \ /etc/systemd/system/default.target.wants/systemd-user-sessions.service + RUN {[minio-client-install]} + CMD ["/usr/sbin/init"] @@ -544,6 +588,9 @@
Introduction + + +

This user guide is intended to be followed sequentially from beginning to end — each section depends on the last. For example, the Backup section relies on setup that is performed in the Quick Start section. Once is up and running then skipping around is possible but following the user guide in order is recommended the first time through.

Although the examples are targeted at {[user-guide-os]} and {[pg-version]}, it should be fairly easy to apply this guide to any Unix distribution and version. The only OS-specific commands are those to create, start, stop, and drop clusters. The commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary. @@ -720,14 +767,6 @@

Installation - - - - - echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts - - -

A new host named pg1 is created to contain the demo cluster and run examples.

@@ -1985,11 +2024,20 @@

supports locating repositories in S3-compatible object stores. The bucket used to store the repository must be created in advance — will not do it automatically. The repository can be located in the bucket root (/) but it's usually best to place it in a subpath so object store logs or other data can also be stored in the bucket without conflicts.

- + Create the bucket + + + echo "{[host-s3-ip]} {[s3-bucket]}.{[s3-endpoint]} {[s3-endpoint]}" | tee -a /etc/hosts + + - aws s3 --no-verify-ssl mb s3://demo-bucket 2>&1 + mc config host add demo https://{[host-s3-ip]} {[s3-key]} {[s3-key-secret]} --insecure + + + + mc mb demo/{[s3-bucket]} --insecure @@ -2004,18 +2052,18 @@ Configure <proper>S3</proper> s3 - /demo-repo - accessKey1 - verySecretKey1 - demo-bucket - s3.amazonaws.com - us-east-1 - n + /{[s3-repo]} + {[s3-key]} + {[s3-key-secret]} + {[s3-bucket]} + {[s3-endpoint]} + {[s3-region]} + /etc/pki/tls/certs/ca-bundle.crt 4 - The region and endpoint will need to be configured to where the bucket is located. The values given here are for the us-east-1 region. + The region and endpoint will need to be configured to where the bucket is located. The values given here are for the {[s3-region]} region.

A role should be created to run and the bucket permissions should be set as restrictively as possible. This sample Amazon S3 policy will restrict all reads and writes to the bucket and repository path.

@@ -2029,13 +2077,13 @@ "s3:ListBucket" ], "Resource": [ - "arn:aws:s3:::demo-bucket" + "arn:aws:s3:::{[s3-bucket]}" ], "Condition": { "StringEquals": { "s3:prefix": [ "", - "demo-repo" + "{[s3-repo]}" ], "s3:delimiter": [ "/" @@ -2049,12 +2097,12 @@ "s3:ListBucket" ], "Resource": [ - "arn:aws:s3:::demo-bucket" + "arn:aws:s3:::{[s3-bucket]}" ], "Condition": { "StringLike": { "s3:prefix": [ - "demo-repo/*" + "{[s3-repo]}/*" ] } } @@ -2067,7 +2115,7 @@ "s3:DeleteObject" ], "Resource": [ - "arn:aws:s3:::demo-bucket/demo-repo/*" + "arn:aws:s3:::{[s3-bucket]}/{[s3-repo]}/*" ] } ] @@ -2079,11 +2127,6 @@ Create the stanza - - - echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts - - {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-create completed successfully diff --git a/test/Vagrantfile b/test/Vagrantfile index fd867467c..9173ea567 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -63,22 +63,6 @@ Vagrant.configure(2) do |config| apt-get install -y devscripts build-essential lintian git lcov cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \ libxml2-dev liblz4-dev - #--------------------------------------------------------------------------------------------------------------------------- - echo 'Install AWS CLI' && date - apt-get install -y python-pip - pip install --upgrade awscli - - # Configure AWS CLI - sudo -i -u vagrant aws configure set region us-east-1 - sudo -i -u vagrant aws configure set aws_access_key_id accessKey1 - sudo -i -u vagrant aws configure set aws_secret_access_key verySecretKey1 - - # Create test alias for AWS CLI - echo '' >> /home/vagrant/.profile - echo '# Test alias for AWS CLI' >> /home/vagrant/.profile - echo 'alias s3-test="export PYTHONWARNINGS=ignore && aws s3 --endpoint-url=https://172.17.0.2 --no-verify-ssl"' \ - >> /home/vagrant/.profile - #--------------------------------------------------------------------------------------------------------------------------- echo 'Install Devel::Cover' && date dpkg -i /backrest/test/package/u18-libdevel-cover-perl_1.29-2_amd64.deb diff --git a/test/lib/pgBackRestTest/Common/HostGroupTest.pm b/test/lib/pgBackRestTest/Common/HostGroupTest.pm index dd09bad26..9243be45a 100644 --- a/test/lib/pgBackRestTest/Common/HostGroupTest.pm +++ b/test/lib/pgBackRestTest/Common/HostGroupTest.pm @@ -59,19 +59,17 @@ sub hostAdd $strOperation, $oHost, $rstryHostName, - $bUpdateHosts, ) = logDebugParam ( __PACKAGE__ . '->hostAdd', \@_, {name => 'oHost'}, {name => 'rstryHostName', optional => true}, - {name => 'bUpdateHosts', default => true, optional => true}, ); $self->{host}{$oHost->{strName}} = $oHost; - if ($bUpdateHosts) + if ($oHost->hostUpdateGet()) { $oHost->executeSimple("echo \"\" >> /etc/hosts", undef, 'root', {bLoadEnv => false}); $oHost->executeSimple("echo \"# Test Hosts\" >> /etc/hosts", undef, 'root', {bLoadEnv => false}); @@ -87,10 +85,14 @@ sub hostAdd if ($strOtherHostName ne $oHost->{strName}) { # Add this host IP to all hosts - $oOtherHost->executeSimple("echo \"$oHost->{strIP} ${strHostList}\" >> /etc/hosts", undef, 'root', {bLoadEnv => false}); + if ($oOtherHost->hostUpdateGet()) + { + $oOtherHost->executeSimple( + "echo \"$oHost->{strIP} ${strHostList}\" >> /etc/hosts", undef, 'root', {bLoadEnv => false}); + } # Add all other host IPs to this host - if ($bUpdateHosts) + if ($oHost->hostUpdateGet()) { $oHost->executeSimple( "echo \"$oOtherHost->{strIP} ${strOtherHostName}\" >> /etc/hosts", undef, 'root', {bLoadEnv => false}); diff --git a/test/lib/pgBackRestTest/Common/HostTest.pm b/test/lib/pgBackRestTest/Common/HostTest.pm index 66e1378ff..9a1e23e1e 100644 --- a/test/lib/pgBackRestTest/Common/HostTest.pm +++ b/test/lib/pgBackRestTest/Common/HostTest.pm @@ -41,6 +41,7 @@ sub new $self->{stryMount}, $self->{strOption}, $self->{strParam}, + $self->{bHostUpdate}, ) = logDebugParam ( @@ -53,6 +54,7 @@ sub new {name => 'stryMount', required => false, trace => true}, {name => 'strOption', required => false, trace => true}, {name => 'strParam', required => false, trace => true}, + {name => 'bHostUpdate', required => false, trace => true, default => true}, ); executeTest("docker rm -f $self->{strContainer}", {bSuppressError => true}); @@ -250,6 +252,16 @@ sub copyFrom return logDebugReturn($strOperation); } +#################################################################################################################################### +# hostUpdateGet +#################################################################################################################################### +sub hostUpdateGet +{ + my $self = shift; + + return $self->{bHostUpdate}; +} + #################################################################################################################################### # ipGet ####################################################################################################################################