diff --git a/test/Vagrantfile b/test/Vagrantfile index e3727e3f1..34856db91 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -70,16 +70,6 @@ Vagrant.configure(2) do |config| apt-get install -y devscripts build-essential lintian git lcov cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \ libxml2-dev liblz4-dev libpq-dev valgrind - #--------------------------------------------------------------------------------------------------------------------------- - echo 'Install AWS CLI' && date - apt-get install -y python-pip - pip install --upgrade awscli - - # Configure AWS CLI - sudo -i -u vagrant aws configure set region us-east-1 - sudo -i -u vagrant aws configure set aws_access_key_id accessKey1 - sudo -i -u vagrant aws configure set aws_secret_access_key verySecretKey1 - #--------------------------------------------------------------------------------------------------------------------------- echo 'Install Docker' && date curl -fsSL https://get.docker.com | sh diff --git a/test/docker/s3-server.docker b/test/docker/s3-server.docker deleted file mode 100644 index a28aedc4e..000000000 --- a/test/docker/s3-server.docker +++ /dev/null @@ -1,50 +0,0 @@ -# S3 Server used for testing and documentation -# docker build -f s3-server.docker -t pgbackrest/test:s3-server-20180612A . -FROM ubuntu:18.04 - -# Suppress dpkg interactive output -RUN rm /etc/apt/apt.conf.d/70debconf && \ - -# Install base packages - apt-get update && \ - apt-get -y install build-essential openssl wget git python-pip && \ - -# Fix root tty - sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile && \ - -# Generate fake certs - mkdir -p -m 755 /etc/fake-cert && \ - cd /etc/fake-cert && \ - openssl genrsa -out ca.key 2048 && \ - openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 \ - -subj "/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org" && \ - openssl genrsa -out server.key 2048 && \ - openssl req -new -key server.key -out server.csr \ - -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org" && \ - openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 \ - -sha256 && \ - chmod 644 /etc/fake-cert/* && \ - -# Install AWS CLI - pip install --upgrade awscli && \ - aws configure set region us-east-1 && \ - aws configure set aws_access_key_id accessKey1 && \ - aws configure set aws_secret_access_key verySecretKey1 && \ - -# Install node.js - wget -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x && \ - bash /root/nodejs.sh && \ - apt-get install -y nodejs npm && \ - -# Install Scality S3 - wget -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz && \ - mkdir /root/scalitys3 && \ - tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz && \ - cd /root/scalitys3 && \ - npm install && \ - sed -i "0,/,/s//,\n \"certFilePaths\":{\"key\":\"\/etc\/fake\-cert\/server.key\",\"cert\":\"\/etc\/fake\-cert\/server.crt\",\"ca\":\"\/etc\/fake\-cert\/ca.crt\"},/" \ - ./config.json && \ - sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json - -# Start SSH when container starts -ENTRYPOINT npm start --prefix /root/scalitys3 diff --git a/test/lib/pgBackRestTest/Common/FileTest.pm b/test/lib/pgBackRestTest/Common/FileTest.pm index e2a67a3ce..e446a3275 100644 --- a/test/lib/pgBackRestTest/Common/FileTest.pm +++ b/test/lib/pgBackRestTest/Common/FileTest.pm @@ -32,6 +32,7 @@ use pgBackRest::Storage::Base; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::HostGroupTest; use pgBackRestTest::Common::LogTest; +use pgBackRestTest::Common::RunTest; use pgBackRestTest::Common::VmTest; use pgBackRestTest::Env::Host::HostBaseTest; use pgBackRestTest::Env::Host::HostBackupTest; @@ -204,9 +205,34 @@ sub forceStorageMove # If S3 then use storage commands to remove if ($oStorage->type() eq STORAGE_S3) { - hostGroupGet()->hostGet(HOST_S3)->executeS3( - 'mv' . ($bRecurse ? ' --recursive' : '') . ' s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strSourcePathExp) . - ' s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strDestinationPathExp)); + if ($bRecurse) + { + my $rhManifest = $oStorage->manifest($strSourcePathExp); + + foreach my $strName (sort(keys(%{$rhManifest}))) + { + if ($rhManifest->{$strName}{type} eq 'f') + { + $oStorage->put( + new pgBackRest::Storage::StorageWrite( + $oStorage, + pgBackRest::LibC::StorageWrite->new( + $oStorage->{oStorageC}, "${strDestinationPathExp}/${strName}", 0, undef, undef, 0, true, false)), + ${$oStorage->get( + new pgBackRest::Storage::StorageRead( + $oStorage, + pgBackRest::LibC::StorageRead->new( + $oStorage->{oStorageC}, "${strSourcePathExp}/${strName}", false)))}); + + $oStorage->remove("${strSourcePathExp}/${strName}"); + } + } + } + else + { + $oStorage->put($strDestinationPathExp, ${$oStorage->get($strSourcePathExp)}); + $oStorage->remove($strSourcePathExp); + } } # Else remove using filesystem commands else diff --git a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm index 3a2fe6fca..3557c3437 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm @@ -85,48 +85,4 @@ sub new ); } -#################################################################################################################################### -# executeS3 -#################################################################################################################################### -sub executeS3 -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strCommand - ) = - logDebugParam - ( - __PACKAGE__ . '->executeS3', \@_, - {name => 'strCommand', trace => true}, - ); - - # Retry the command until timeout - my $oWait = waitInit(60); - my $bSuccess = false; - my $strTotalCommand = - 'export PYTHONWARNINGS="ignore" && aws --endpoint-url=https://' . $self->ipGet() . ' s3 --no-verify-ssl ' . $strCommand; - - do - { - my $oExec = new pgBackRestTest::Common::ExecuteTest($strTotalCommand, {bSuppressError => true, bSuppressStdErr => true}); - $oExec->begin(); - - $bSuccess = $oExec->end() == 0; - } - while (!$bSuccess && waitMore($oWait)); - - # If no success run again to display the error - if (!$bSuccess) - { - executeTest($strTotalCommand); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - 1; diff --git a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm index 0d7ed6bd8..214ec94cc 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm @@ -540,7 +540,7 @@ sub run if ($bS3) { - $oHostS3->executeS3('cp /etc/hosts s3://' . HOST_S3_BUCKET . "${strTempFile}"); + storageRepo()->put($strTempFile, "TEMP"); } else { diff --git a/test/travis.pl b/test/travis.pl index 6bafa2549..cade1e22b 100755 --- a/test/travis.pl +++ b/test/travis.pl @@ -168,7 +168,7 @@ eval } else { - $strPackage .= " python-pip libdbd-pg-perl"; + $strPackage .= " libdbd-pg-perl"; } processBegin('install test packages'); @@ -191,13 +191,6 @@ eval processExec("sudo adduser --ingroup=\${USER?} --uid=5001 --disabled-password --gecos \"\" " . BACKREST_USER); processEnd(); - processBegin("install and configure aws cli"); - processExec('pip install --upgrade --user awscli', {bSuppressStdErr => true}); - processExec('aws configure set region us-east-1'); - processExec('aws configure set aws_access_key_id accessKey1'); - processExec('aws configure set aws_secret_access_key verySecretKey1'); - processEnd(); - # Build the container processBegin("${strVm} build"); processExec("${strTestExe} --vm-build --vm=${strVm}", {bShowOutputAsync => true, bOutLogOnError => false});